@cloudbitmaps/core 0.11.2 → 0.13.0

package/dist/core/crbm-storage-source.d.ts

@@ -1,11 +1,12 @@
 import { type IAuditSink } from './audit.js';
 import type { Yielder } from './cooperative.js';
-import type { Clock } from './determinism.js';
+import type { Clock, Rng } from './determinism.js';
 import type { CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
-import type { ChunkRef, StorageChunkSource, GenKey, IStorageDriver, IRegistryDriver, SegmentRef, SegmentSize, Token } from './ports.js';
+import type { ChunkRef, StorageChunkSource, GenerationMetadata, GenerationSummary, GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, RegistrySummary, SegmentRef, SegmentSize, Token } from './ports.js';
 import { CrbmReader } from './crbm/reader.js';
 import type { CrbmReaderOptions } from './crbm/reader.js';
 import type { CodecBitmap, CodecInterface } from './codec.js';
+import { DecodedLoadInput } from './load-input.js';
 export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
     /**
      * Optional {@link IRegistryDriver}. When provided, the current generation is resolved via the registry's
@@ -57,8 +58,8 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
      * Aggregate byte ceiling on the parsed `.crbm` indices resident in the reader cache (default 64 MiB) — the
      * **second half of that memory bound**. `maxOpenSegments` alone bounds by *count*, but a wide/dense segment's
      * parsed index can reach about 1.3 MB, so 1024 wide indices could pin over a GB and blow a small heap (e.g. a 128 MB
-     * Lambda) while the count is nominally "in bounds". This caps the summed {@link CrbmReader.retainedIndexBytes}
-     * across cached readers; the least-recently-used reader is evicted once the total would exceed it — whichever
+     * Lambda) while the count is nominally "in bounds". This caps the summed {@link CrbmReader.retainedBytes} (the
+     * parsed index, and any metadata a generation carries) across cached readers; the least-recently-used reader is evicted once the total would exceed it — whichever
      * of the count/byte bounds binds first. Lower it for memory-tight deployments with wide segments; a single
      * segment whose index alone exceeds the budget is still cached (it can't be shrunk) but nothing else alongside.
      */
@@ -110,6 +111,13 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
      * check again.
      */
     private invalidations;
+    /**
+     * Generations whose row summary this process found to disagree with their object, by segment and generation. A
+     * summary here is not used again: a count of that generation reads the object. Bounded as the readers are, so a
+     * long-lived process cannot grow it without limit; one that falls out is trusted again until its next open finds
+     * the disagreement again.
+     */
+    private readonly distrusted;
     private readonly registry;
     private readonly keystore;
     private readonly requireEncryption;
@@ -117,7 +125,12 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
     private readonly clock;
     private readonly currentGenTtlMs;
     constructor(driver: IStorageDriver, options?: CrbmStorageChunkSourceOptions);
-    /** The current resolved reader for a segment, refreshing on the TTL. Cheap within the TTL window. */
+    /**
+     * The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window. The snapshot holds the resolved
+     * target, and its reader once a read has asked for one.
+     */
+    private liveSnapshot;
+    /** The current resolved reader for a segment, refreshing on the TTL. */
     private resolvedReader;
     /**
      * A refresh failed transiently and the prior reader keeps serving: make this snapshot lapse after
@@ -125,12 +138,16 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
      * that asks the registry again. It costs no read of its own; it only changes when the next refresh happens.
      */
     private retrySoon;
-    /** TTL elapsed: cheaply re-resolve `currentGen`; reopen only if it actually changed, else reuse the prior reader. */
-    private computeRefreshedReader;
     /**
-     * Memoize a reader-promise as the segment's snapshot at `now()`, forgetting it later if it resolves to
-     * null / throws (identity-guarded, so a stale cleanup never clobbers a fresher snapshot). Installing the
-     * (pending) promise **synchronously** is what lets concurrent callers coalesce onto one in-flight resolve.
+     * TTL elapsed: cheaply re-resolve the pointer. The row read is always fresh, so what the new snapshot says of the
+     * generation (its summary included) is what the row says now, never the prior snapshot's. The reader is reopened
+     * only if the generation or the row changed, and only when a read asks for it.
+     */
+    private refreshedTarget;
+    /**
+     * Memoize a snapshot as the segment's at `now()`, forgetting it later if it resolves to null / throws
+     * (identity-guarded, so a stale cleanup never clobbers a fresher snapshot). Installing the (pending) snapshot
+     * **synchronously** is what lets concurrent callers coalesce onto one in-flight resolve.
      */
     private install;
     private now;
@@ -273,26 +290,51 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
      */
     exists(ref: SegmentRef): Promise<boolean>;
     /**
-     * Evict a snapshot we just failed to read from — matched by its reader *promise*, so a concurrent call's
+     * Evict a snapshot we just failed to read from — matched by the snapshot itself, so a concurrent call's
      * fresher snapshot is never clobbered. The identity guard is the whole point: `install` replaces the entry
      * wholesale, so comparing anything else would drop a good snapshot on the floor.
      */
     private dropStale;
-    private resolveLatest;
     /** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
     private openForTarget;
+    private openGeneration;
+    /**
+     * Open the generation a resolution found, and hold the row's summary against the object while it is open anyway:
+     * the same count and metadata, or the summary is not believed again in this process ({@link distrusted}). This
+     * costs no request; a mismatch is not an error, since the object is the truth and this read has it.
+     */
+    private openLive;
+    private distrustKey;
+    /** What the row says of the generation, unless it is a summary this process has found to disagree with its object. */
+    private trustedSummary;
     /**
      * The current generation + its DEK wrappings: the registry's authoritative record (one strong read), or a
      * `list` scan for the max generation when there's no registry (a registry-less source can only read cleartext
-     * — there's nowhere a wrapped DEK could live).
+     * — there's nowhere a wrapped DEK could live). The row's summary rides along, to be used or not by whoever
+     * asks.
      */
-    private resolveTarget;
+    private resolveLive;
     /** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
     private cryptoForRead;
     getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
     listChunkKeys(ref: SegmentRef): Promise<number[]>;
     sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
     cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
+    /**
+     * The current generation's number, id count and metadata. The answer comes from the row the snapshot resolved when
+     * it can: the row's summary of the generation it names, on an active row, in the shape the row's keys call for
+     * (a sealed one is opened with the segment's key), and one this process has not found to disagree with its
+     * object. That is one registry read when the snapshot is cold, none when it is warm, and no read of the object.
+     * With no summary it can use (an older row, one for another generation, one that does not open), it opens the
+     * generation and answers from the object. A row that names an object that is gone still answers, while a read
+     * of the object throws: the number is true of the generation the row names.
+     *
+     * What the row says is not confirmed here: a summary edited to agree with itself by whoever can write the row is
+     * believed until the next time the object is opened, which holds it against the summary.
+     */
+    summary(ref: SegmentRef): Promise<GenerationSummary | null>;
+    /** {@link summary} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
+    summaryAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<GenerationSummary | null>;
     /**
      * Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
      * generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
@@ -320,6 +362,17 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
      * `intersect` would do it N times. Gated in `tests/core/storage-source-heal-open.test.ts` by counting calls.
      */
     private withFreshSnapshot;
+    /**
+     * Whether `err`, raised by a read through the snapshot's reader, came from an object that is no longer the one the
+     * reader opened. A generation number can be taken again once its object is deleted (an erasure removes the
+     * generations above a rolled-back pointer that held the id, a collection pass can stop part-way, and a load then
+     * numbers `currentGen + 1` again), so a reader that still holds the old object's index reads the new object's
+     * bytes at the old offsets: a chunk checksum fails, or a range runs past the end. The object's footer says which it
+     * was, as it does for a pin ({@link CrbmReader.sameObject}): replaced, or gone, re-resolves; the same object means
+     * the error is real, and it stands. Only these two errors pay for the footer read, never a read that succeeded. A
+     * transient fault in the check is rethrown; a footer that cannot be read says nothing, and the error stands.
+     */
+    private replacedUnder;
 }
 /**
  * Write one immutable generation from in-memory bitmaps. Chunks are
@@ -333,9 +386,30 @@ export declare function writeCrbmGeneration(driver: IStorageDriver, key: GenKey,
 }>, options?: {
     crypto?: CrbmCrypto;
     clock?: Yielder;
+    metadata?: GenerationMetadata;
 }): Promise<{
     size: number;
     sha256: string;
+    fingerprint: string;
+}>;
+/**
+ * Whether the object under `key` is provably the one `fingerprint` names ({@link CrbmReader.fingerprint}), from one
+ * read of its footer. False when it is another object, when it is gone, and when the read fails in any way: a caller
+ * that deletes on `true` deletes nothing it cannot prove is its own.
+ */
+export declare function holdsObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
+/** Whether the object under `key` says it is encrypted, from one read of its footer, with no key. */
+export declare function objectIsEncrypted(storage: IStorageDriver, key: GenKey): Promise<boolean>;
+/**
+ * What one tail read of the object under `key` tells a rollback about it: whether the footer says it is encrypted, and,
+ * when the object and the key agree (an encrypted object opened with `crypto`, a cleartext one with none), the reader
+ * the open gives, which holds its count and metadata. No reader when they disagree, which the caller refuses as a
+ * target the row cannot use, and none for an encrypted object when the caller has no key. A footer that fails its own
+ * checks, an object another generation's, and an index or metadata that do not open are an {@link IntegrityError}.
+ */
+export declare function openRollbackTarget(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined): Promise<{
+    encrypted: boolean;
+    reader: CrbmReader | undefined;
 }>;
 /** What {@link writeCrbmGenerationStream} wrote: the driver's `{ size, sha256 }` + a tally of the generation. */
 export interface StreamWriteResult {
@@ -345,12 +419,14 @@ export interface StreamWriteResult {
     readonly chunkKeys: number[];
     /** Total ids written. */
     readonly cardinality: number;
+    /** The fingerprint of what this call wrote ({@link CrbmReader.fingerprint}), which tells it from another object under its key. */
+    readonly fingerprint: string;
 }
 /**
  * Streaming variant of {@link writeCrbmGeneration} for a **constant-memory** rewrite (the erasure rewrite, a
  * sorted load): consumes an **already-ascending** async stream of `{ chunkKey, bitmap }`, feeding each to the
  * codec and freeing it, instead of materializing the whole generation. Paired with a streaming storage sink (S3
- * multipart / LocalFs temp file), peak memory is ~one chunk + one part. Returns a tally so the caller can verify
+ * multipart / LocalFs temp file), the writer itself holds ~one chunk + one part. Returns a tally so the caller can verify
  * the re-opened object without re-iterating the (now-consumed) stream. Input **must** be ascending by `chunkKey`
  * — the codec rejects an out-of-order chunk; empty bitmaps are skipped.
  */
@@ -360,6 +436,7 @@ export declare function writeCrbmGenerationStream(driver: IStorageDriver, key: G
 }>, options?: {
     crypto?: CrbmCrypto;
     clock?: Yielder;
+    metadata?: GenerationMetadata;
 }): Promise<StreamWriteResult>;
 /**
  * Point a segment's registry `currentGen` at `key.generation` — the publish step that makes a freshly-written
@@ -385,17 +462,43 @@ export declare function writeCrbmGenerationStream(driver: IStorageDriver, key: G
  * so the check and the write see the same row.
  *
  * **`expectToken` fences the LINEAGE, and a derived writer needs both.** A generation number identifies a
- * generation only within one incarnation of a name: `nextGeneration` restarts at `0` once the row is purged and
+ * generation only within one incarnation of a name: the numbering restarts at `0` once the row is purged and
  * the bucket empty, so a name that is retired and re-created has a *different* segment wearing the *same*
  * `currentGen`. `expectFrom` alone matched it — and an erasure rewrite then published one incarnation's content
  * over another's, deleted the live objects with its `keep: 0` collection, and returned `erased: true`. The row's
- * OCC token is the identity that survives this: {@link IRegistryDriver.delete} tombstones rather than unlinks,
- * so "a later `create` still gets a fresh, greater token" and no token is ever reused across incarnations.
+ * OCC token is the identity that survives this: a shipped registry's later `create` gets a token never issued before
+ * under that name, but for a collision of probability 2^-128 per pair of incarnations, so no token is reused across them.
  *
  * Pass the token read alongside `expectFrom` and the publish lands only on the same row it was derived from.
  * The check is deliberately **conservative**: the token also changes on writes that are not supersessions at
  * all (a `setRetention`, a due-index reindex), so such a write makes a derived publish report `superseded` and
  * the caller re-derive. That costs a re-run on a rare, unrelated write; the alternative costs a segment.
+ *
+ * **The driver sends a registry write once, and an unanswered one is reconciled by its effect.** A `create` or
+ * `compareAndSwap` that fails with {@link TransientError} (a throttle, a lost response, a timeout) may or may not
+ * have landed, and may still be in flight. The publish reads the row once more and decides from what it holds:
+ *
+ * - the pointer names `key.generation`, on the incarnation the write was made against (the row's `createdAt`), and
+ *   `holdsOwnObject` proves the object under it the caller's: the write landed, and the publish returns `true`. Only
+ *   the writer of a write-once object publishes its number, and a number is never taken while an object holds it;
+ * - the row is still the one the write was made against (the same token, or still no row): the write may land yet. The
+ *   publish then waits, on the injected clock, and sends a **new** write from the row it just read: a request of its
+ *   own, not a replay of the driver's, carrying the same version, so under the registry's fence at most one of the two
+ *   lands and the other meets a conflict, which the next read settles. At most {@link UNANSWERED_RESENDS} follow, after
+ *   a wait under 500 ms, then 1 s, then 2 s (full jitter when an `rng` is given); each is settled the same way. Still
+ *   unanswered, or with no clock to wait on, this throws the registry's own `TransientError` and writes nothing more.
+ *   A caller must not delete its object on it;
+ * - anything else: the row has moved past the state the write was conditioned on, so that write can never land, and
+ *   the publish goes on as after a lost race, from the row it just read. A pointer at `key.generation` that is not over
+ *   the caller's own object, or is on another incarnation, is a lost race too and answers `false`, never an already
+ *   current re-publish.
+ *
+ * A failed read throws `TransientError` too. The same check of the pointer runs after a {@link WriteConflictError},
+ * so a write that landed and then met itself (a registry whose client re-sent it) is not taken for a lost race.
+ *
+ * **A `summary` goes in the same write that moves the pointer**, so a reader that sees this generation as current
+ * sees the count and metadata that describe it, and every attempt sends the same one. It must be the shape the row's
+ * keys call for: sealed for an object written with a key, clear for one written without.
  */
 export declare function publishGeneration(registry: IRegistryDriver, key: GenKey, options?: {
     wrappedDeks?: readonly WrappedDek[];
@@ -417,7 +520,55 @@ export declare function publishGeneration(registry: IRegistryDriver, key: GenKey
      * destination that does not exist yet is the ordinary first run of a pipeline.
      */
     expectAbsent?: boolean;
+    /**
+     * The generation's summary (see {@link RegistrySummary}), written by the write that moves the pointer to it. It
+     * names `key.generation`. Absent, the row is left with none for this generation.
+     */
+    summary?: RegistrySummary;
+    /**
+     * The segment's row as the caller already read it (`null`: it found none). The first attempt acts on it
+     * instead of reading the row again, which is sound because that attempt writes only under the row's own
+     * fence: its compare-and-swap carries this row's token, and with no row it creates create-only, so a row
+     * that changed or appeared since makes the write lose and the next attempt reads the row afresh. Only the
+     * idempotent "already current" answer writes nothing, so a reused row never gives it.
+     */
+    row?: RegistryRecord | null;
+    /**
+     * The object was written without encryption. Its publish is refused, with {@link KeyUnavailableError}, onto a row
+     * that carries key material: a reader of that segment opens every generation with its key, and a later
+     * `destroySegment` would attest that shredding it made a readable object unreadable. A caller decides its key
+     * from the row it read before the write, so this closes the window in which another writer creates the
+     * segment encrypted meanwhile.
+     */
+    cleartext?: boolean;
+    /**
+     * Whether the object under `key` is the one the caller wrote, and throws when it cannot tell. Asked only when a
+     * registry write failed and the row then names `key.generation`, to tell this publish's write from another
+     * writer's at the same number. Every caller that writes the object passes it. Without it, a write that ended
+     * without an answer over a row that names `key.generation` cannot be settled: the publish throws
+     * {@link TransientError} and the caller keeps its object.
+     */
+    holdsOwnObject?: () => Promise<boolean>;
+    /**
+     * What a publish waits on before it sends a fresh compare-and-swap, after a write that ended without an answer and
+     * left the row as it was. Without a `sleep`, such a write throws at once instead of being sent again.
+     */
+    clock?: Yielder;
+    /** Spreads that wait (full jitter) so concurrent publishers do not retry together. Without it the wait is the bound. */
+    rng?: Rng;
+    /**
+     * Called once for each registry write that ended without an answer, before the row is read back. A caller whose
+     * publish then answers `false` can say that it cannot tell whether its own write landed first: the row may have
+     * held its generation for a while before another writer moved on.
+     */
+    onUnanswered?: () => void;
 }): Promise<boolean>;
+/**
+ * Whether the object under `key` is the one `fingerprint` names, from one read of its footer: false when it is another
+ * object or gone. Unlike {@link holdsObject}, a read that cannot tell throws, for a caller that must not take "could
+ * not look" for "not this object".
+ */
+export declare function provesOwnObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
 /** What a bulk-load wrote — the driver's `{ size, sha256 }` plus a summary of the built generation. */
 export interface BulkLoadResult {
     /** Bytes written to the Storage object. */
@@ -436,8 +587,18 @@ export interface BulkLoadResult {
      * unreadable: the bytes are encrypted with a key nothing recorded.
      */
     readonly wrappedDeks?: readonly WrappedDek[];
+    /** Whether the object was written encrypted: under the segment's key, or under one this call minted. */
+    readonly encrypted: boolean;
+    /** The written object's fingerprint, which tells it from an object stored later under the same key. */
+    readonly fingerprint: string;
     /** Total distinct ids in the generation (post-dedup). */
     readonly cardinality: number;
+    /**
+     * The summary of what was written, for the write that makes it current: its id count and metadata, sealed under the
+     * segment's key when the object was written with one. A caller that defers the publish (`publish: false`) passes it
+     * to {@link publishGeneration}.
+     */
+    readonly summary: RegistrySummary;
     /**
      * Whether this generation is now the segment's **current** one — `undefined` when no `registry` was wired, so
      * there is no pointer and no publish step to report on.
@@ -467,11 +628,12 @@ export interface BulkLoadResult {
  * commits only after the callback resolves). An empty source writes a valid empty generation.
  *
  * Writes a fresh full snapshot of a segment. The caller picks the generation number in `key` — `nextGeneration`
- * computes the right one from the registry and the bucket — and re-using an existing generation throws
+ * computes one from the registry and the bucket, as `nextLoadGeneration` does for a load — and re-using an existing
+ * generation throws
  * {@link WriteConflictError} (write-once). Without a registry a `StorageChunkSource` serves the **highest**
  * generation present, so a too-high number silently shadows real data; with one, `publishGeneration` decides.
  */
-export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenKey, ids: Iterable<number> | AsyncIterable<number>, options?: {
+export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenKey, ids: Iterable<number> | AsyncIterable<number> | DecodedLoadInput, options?: {
     registry?: IRegistryDriver;
     keystore?: IKeystore;
     requireEncryption?: boolean;
@@ -502,6 +664,22 @@ export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenK
      * current.
      */
     publish?: boolean;
+    /**
+     * The generation's metadata (see {@link GenerationMetadata}), written into the object and carried in the summary.
+     * The caller has checked it against the metadata rules; the writer checks it again, and refuses a record that breaks
+     * one before a byte is written.
+     */
+    metadata?: GenerationMetadata;
+    /**
+     * The segment's row as the caller already read it (`null`: it found none), for a caller that defers the
+     * publish (`publish: false`) and fences it on that same row. A present cleartext row is then used as read,
+     * rather than read again: a row that changes in between (a publish, a drop, a purge and re-create) makes the
+     * fenced publish lose, so the object is never published on the strength of the stale read. A row read as
+     * absent, or one carrying key material, is read again after the ids, as with no row passed: a first load must
+     * see a row another writer created meanwhile, and an encrypted segment's key is unwrapped only from a row read
+     * after the ids, so a segment shredded while they streamed is refused before its key is used.
+     */
+    row?: RegistryRecord | null;
 }): Promise<BulkLoadResult>;
 /**
  * Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
@@ -511,12 +689,15 @@ export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenK
 export declare function openGenerationReader(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, options?: Omit<CrbmReaderOptions, 'crypto'>): Promise<CrbmReader>;
 /**
  * Re-open a freshly written generation and assert it round-trips exactly what was streamed into it: the same
- * per-chunk key set *and* the same total cardinality (on top of the codec's own per-chunk CRC + footer checks).
- * `expected` is the streaming writer's tally (the stream is consumed, so it can't be re-iterated) — the key-set
- * comparison catches a dropped/extra chunk that a cardinality-only check could miss when two errors cancel out.
+ * per-chunk key set *and* the same total cardinality (on top of the codec's own per-chunk CRC + footer checks), and the
+ * metadata it was given. `expected` is the streaming writer's tally (the stream is consumed, so it can't be
+ * re-iterated) — the key-set comparison catches a dropped/extra chunk that a cardinality-only check could miss when
+ * two errors cancel out. The count and the metadata are what the generation's row summary is about to claim, so the
+ * summary is held to the object it describes before it is published.
  * Throws {@link IntegrityError}: the object is on disk but must not be published.
  */
 export declare function verifyGeneration(storage: IStorageDriver, key: GenKey, expected: {
     readonly chunkKeys: readonly number[];
     readonly cardinality: number;
+    readonly metadata?: GenerationMetadata | undefined;
 }, crypto: CrbmCrypto | undefined): Promise<void>;

package/dist/core/crypto.d.ts

@@ -62,19 +62,23 @@ export interface IKeystore {
 }
 /**
  * What the encrypting codec needs: the {@link Aead} for this segment's DEK plus a way to build the AAD for a
- * given scope (`'index'` or a chunkKey). Built per (segment, generation) by the storage-source bridge so the
- * codec itself stays segment-agnostic.
+ * given scope (a chunkKey, `'index'`, or `'metadata'` for the generation's metadata section). Built per
+ * (segment, generation) by the storage-source bridge so the codec itself stays segment-agnostic.
  */
 export interface CrbmCrypto {
     readonly aead: Aead;
-    aadFor(scope: number | 'index'): Uint8Array;
+    aadFor(scope: number | 'index' | 'metadata'): Uint8Array;
 }
 /**
- * Build the AEAD associated-data binding a chunk (or the index) to its exact location:
- * `v1 ‖ len(namespace) ‖ namespace ‖ len(segment) ‖ segment ‖ generation ‖ scope ‖ chunkKey`. Length-prefixed
- * so distinct `(namespace, segment)` pairs can never collide. Pure (no crypto) — just the authenticated label.
+ * Build the AEAD associated-data binding a chunk, the index, the metadata section or a registry row's sealed summary
+ * to its exact location:
+ * `v1 ‖ len(namespace) ‖ namespace ‖ len(segment) ‖ segment ‖ generation ‖ scope ‖ chunkKey`, where scope is
+ * `0` for a chunk, `1` for the index, `2` for the metadata and `3` for the row's summary, and chunkKey is `0` for
+ * the last three.
+ * Length-prefixed so distinct `(namespace, segment)` pairs can never collide. Pure (no crypto) — just the
+ * authenticated label.
  */
 export declare function aadFor(ref: {
     readonly namespace?: string;
     readonly segment: string;
-}, generation: number, scope: number | 'index'): Uint8Array;
+}, generation: number, scope: number | 'index' | 'metadata' | 'summary'): Uint8Array;

package/dist/core/determinism.d.ts

@@ -40,3 +40,10 @@ export interface Rng {
     /** A float in the half-open interval [0, 1). */
     next(): number;
 }
+/**
+ * Injected entropy: `length` cryptographically strong random bytes, for an identity that must never repeat (a
+ * registry row's incarnation id, and the random part every registry write adds to its token). Deliberately not an
+ * {@link Rng}: an `Rng` is seedable for simulation, and a seeded source hands every process the same "random"
+ * identity. Production wiring uses the platform's Web Crypto; a test injects a deterministic source to replay a run.
+ */
+export type Entropy = (length: number) => Uint8Array;

package/dist/core/engine.d.ts

@@ -3,11 +3,12 @@ import type { Budget, BudgetOption } from './budget.js';
 import type { Clock } from './determinism.js';
 import type { BoundedLru } from './lru.js';
 import type { IMetricsSink } from './metrics.js';
-import type { StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
+import type { GenerationMetadata, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
 export interface EngineDeps {
     readonly storage: StorageChunkSource;
     /** Optional cache of decoded (immutable) Storage chunks. */
     readonly cache?: BoundedLru<string, CodecBitmap>;
+    /** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. The `.crbm` reader refuses an entry above its own `maxPayloadBytes` (1 MiB, plus 28 bytes when encrypted) at open, so raise that on the chunk source too. */
     readonly maxBitmapBytes?: number;
     /**
      * The bitmap codec — **required**. `core/` is codec-agnostic: it can have no default, because the concrete codec
@@ -47,7 +48,13 @@ export interface IdRange {
 }
 /** Options common to the chunk-aligned combines. */
 export interface CombineOptions extends IdRange {
-    /** Max chunk keys resolved concurrently — bounds the Storage footprint. A positive integer. */
+    /**
+     * Max chunk keys resolved concurrently — bounds the Storage footprint (about `concurrency × operands` chunks in
+     * flight). A positive integer; default 32. The window opens 8 keys wide (or `concurrency`, if lower) and widens
+     * as keys are taken, so a read that stops early fetches no more than a few keys ahead; one that runs to the end
+     * spends nearly all of its round trips at the full width. A read that stops early has requested up to
+     * `concurrency` keys per operand past the last one it used.
+     */
     readonly concurrency?: number;
     /** Override the store's per-op budget for this call (`false` lifts it). */
     readonly budget?: BudgetOption;
@@ -70,19 +77,37 @@ export declare class SegmentEngine {
     private readonly metricsOn;
     /** Resolved per-op budget (null = disabled); undefined deps ⇒ the generous default. */
     private readonly budget;
+    /**
+     * The chunk reads open right now, by cache key: a caller that misses the cache and finds its key here awaits that
+     * read instead of making its own. An entry lives from the request until it settles, so the map holds at most one
+     * promise per distinct key in flight.
+     */
+    private readonly openReads;
     constructor(deps: EngineDeps);
     /** Membership: one chunk lookup — the cache, else one Storage fetch of that chunk. */
     has(seg: SegmentRef, id: number): Promise<boolean>;
     /**
-     * Cardinality. When the Storage source can serve per-chunk cardinality from its `.crbm` index (the `.crbm`
-     * source can), the count is summed from the index with **zero payload reads**. A source without that
-     * capability (the in-memory source) falls back to fetching every chunk.
+     * Cardinality. When the Storage source can describe the current generation (the `.crbm` source can), the count is
+     * that description's: the registry row's summary when the row has one it can use, which is one request when cold
+     * and no read of the object, and otherwise the sum of the object's index. Either way, **zero payload reads**. A
+     * source that can only serve per-chunk cardinality sums those, and one with neither (the in-memory source) falls back
+     * to fetching every chunk.
      *
-     * The index sum is trusted, not confirmed: the reader checked the index for internal consistency when it opened
+     * The row's summary and the index sum are trusted, not confirmed: the source holds the summary against the object
+     * whenever it opens it, and the reader checked the index for internal consistency when it opened
      * the object, and a corrupt index that is still internally consistent yields a wrong count. `iterate()` and the
      * combines decode the payloads.
      */
     count(seg: SegmentRef): Promise<number>;
+    /**
+     * What the segment's current generation is: its number, its id count and its metadata, from one resolution, so the
+     * three cannot straddle a publish. A source with no summary answers from `count()` and the generation it resolves.
+     */
+    stat(seg: SegmentRef): Promise<{
+        generation: number | null;
+        cardinality: number;
+        metadata?: GenerationMetadata;
+    }>;
     /** Whether the Storage source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
     get supportsStorageSize(): boolean;
     /** How often the Storage source re-reads a pointer while reading (cost reporting); undefined if it does not say. */
@@ -92,7 +117,7 @@ export declare class SegmentEngine {
     /** The chunks of `seg` at `chunkKeys`, read through a window of {@link DEFAULT_INTERSECT_CONCURRENCY}. */
     private chunkWindow;
     /**
-     * Every id, ascending, reading ahead through a window of up to 8 chunk fetches that opens 1, 2, 4, 8 wide, so a
+     * Every id, ascending, reading ahead through a window of up to 32 chunk fetches that opens 1, 2, 4 … 32 wide, so a
      * read that stops early has fetched only a handful of chunks past the one it stopped in; with `range`, only the ids in `(after, through]`, fetching only the
      * chunks the range overlaps (see {@link IdRange}).
      *
@@ -243,7 +268,15 @@ export declare class SegmentEngine {
      * `gen === undefined` ⇒ the source can't report a generation (it only ever serves one) ⇒ the key stays
      * generation-free. Superseded-generation entries age out under the LRU ceiling — no active purge.
      *
-     * The returned instance is **shared** (it may be the cached one): callers read it or clone it, never mutate it.
+     * Concurrent misses of one key share **one** request, with or without a cache: the first caller starts the read,
+     * later ones await it, and the entry goes when the read settles, so a failure is every waiting caller's and the
+     * next call reads again. The key carries the version exactly as the cache's does, so two versions of one chunk
+     * are two reads, never one.
+     *
+     * The returned instance is **shared** (it may be the cached one, and every caller that joined a read gets the same
+     * one): callers read it or clone it, never mutate it.
      */
     private storageChunk;
+    /** One request for one chunk, decoded, range-checked and cached. */
+    private fetchChunk;
 }

package/dist/core/erase-id.d.ts

@@ -6,8 +6,14 @@
  * is what every other write in this library is: a new generation. The current generation is streamed chunk by
  * chunk through the ascending writer — every chunk decoded, range-checked and re-encoded, the one holding the id
  * with that bit cleared — then published fenced on the generation it streamed, and that generation is collected immediately
- * (`keep: 0`), so the bit is **physically gone from the bucket when this returns**. Constant memory: one chunk in
- * flight, never the whole segment.
+ * (`keep: 0`), so the bit is **physically gone from the bucket when this returns**. Bounded memory: a window of 32
+ * chunk reads ahead of the writer, never the whole segment.
+ *
+ * The rewrite is the same generation without one id, so it keeps everything else: the new object carries the source's
+ * metadata as it is, and the row's summary of it, built from what was written, counts one id fewer and holds the same
+ * metadata. Metadata is not scanned for the id. On an encrypted segment, a source object with no metadata block whose
+ * row's sealed summary of it has metadata is rewritten with the row's: the block's presence is not authenticated, so
+ * the authenticated copy decides, and the erasure still goes through.
  *
  * **`erased: true` is a claim about every generation of the segment, not only the one it replaced.** A rollback
  * leaves generations above the pointer that were once current and can be made current again, so a holder can sit
@@ -69,8 +75,10 @@
 import { type IAuditSink } from './audit.js';
 import type { CodecInterface } from './codec.js';
 import type { Yielder } from './cooperative.js';
+import type { Rng } from './determinism.js';
 import type { IKeystore } from './crypto.js';
 import type { IStorageDriver, IRegistryDriver, SegmentRef } from './ports.js';
+import { type ReadRetry } from './retry.js';
 /** What {@link eraseIdFromSegment} needs: the objects, the pointer, the codec, and the key material if encrypted. */
 export interface EraseIdDeps {
     readonly storage: IStorageDriver;
@@ -90,8 +98,20 @@ export interface EraseIdDeps {
     readonly requireEncryption?: boolean;
     /** Supplying a clock that can yield makes the rewrite cooperative, as it makes a load. */
     readonly clock?: Yielder;
-    /** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. */
+    /**
+     * The random source that spreads the waits between the rewrite's fresh compare-and-swaps. Absent, the read retry's
+     * source is used if it has one, and otherwise each wait is its bound.
+     */
+    readonly rng?: Rng;
+    /** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. The `.crbm` reader refuses an entry above its own `maxPayloadBytes` (1 MiB, plus 28 bytes when encrypted) at open, so raise that on the chunk source too. */
     readonly maxBitmapBytes?: number;
+    /**
+     * The store's read retry, for the reads the rewrite makes along the way: the generation it rewrites and each of its
+     * chunks, the read-back that verifies the generation it wrote, and any other generation it checks for the id. A
+     * transient fault on one is run again under it rather than failing the erasure. Absent, each read is made once.
+     * The deletes are not retried, and the rewrite's registry write is settled as a load's is.
+     */
+    readonly readRetry?: ReadRetry;
 }
 export interface EraseIdResult {
     readonly segment: string;

package/dist/core/erasure.d.ts

@@ -159,10 +159,14 @@ export interface DropResult {
  *    cleaned up by re-running; a torn pointer costs correctness and is not self-healing. Given the choice, leak
  *    bytes — but say so: whatever survives the sweep is reported in {@link DropResult.generationsRemaining}.
  *
- * **Why step 2 sweeps more than once.** A load that was already writing its object when the tombstone landed
- * still finishes the write — its publish is then refused, but the object survives, and it holds the complete
- * set. A single list-then-delete misses it entirely. The re-sweep converges because the tombstone *is* a hard
- * fence on **publishing**, so only already-in-flight writes can appear and they are finite.
+ * **Why step 2 sweeps more than once.** A load that had read the row before the tombstone landed, whether it was
+ * already writing its object or still consuming its ids, still finishes the write — its publish is then refused,
+ * but the object exists, and it holds the complete set. A single list-then-delete misses it entirely. The re-sweep
+ * converges because the tombstone *is* a hard fence on **publishing**, so only loads already under way can write
+ * and they are finite. One that writes after the last pass deletes its own object when its publish is refused, as a
+ * refused load does under a `destroyed` row; only one whose process stops in between, or whose publish fails without
+ * a definite answer (a lost response, a timeout), leaves it behind, for a re-run of the drop, and
+ * `generationsRemaining` cannot report an object written after this call returned.
  *
  * **When "reads as empty" starts being true.** Not instantly, for a store that has already read this segment: a
  * resolved generation is cached and decoded chunks sit in the cache, so an in-flight reader can answer from