@cloudbitmaps/core 0.8.2 → 0.10.0

package/dist/core/crbm-storage-source.d.ts

@@ -0,0 +1,432 @@
+/**
+ * Bridges the `.crbm` archive format to the engine's Storage seam.
+ *
+ * `CrbmStorageChunkSource` implements the {@link StorageChunkSource} the engine reads through, over an
+ * {@link IStorageDriver}: it resolves a segment's current generation, opens its {@link CrbmReader} once, and serves
+ * per-chunk payloads. `writeCrbmGeneration` / `writeCrbmGenerationStream` are the write primitives (a generation
+ * built from bitmaps, in memory or streamed), `bulkLoadCrbmGeneration` is the load path over them, and
+ * `publishGeneration` is the forward-only pointer advance every write ends with.
+ */
+import { type IAuditSink } from './audit.js';
+import type { Yielder } from './cooperative.js';
+import type { Clock } from './determinism.js';
+import type { CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
+import type { ChunkRef, StorageChunkSource, GenKey, IStorageDriver, IRegistryDriver, SegmentRef, SegmentSize, Token } from './ports.js';
+import { CrbmReader } from './crbm/reader.js';
+import type { CrbmReaderOptions } from './crbm/reader.js';
+import type { CodecBitmap, CodecInterface } from './codec.js';
+export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
+    /**
+     * Optional {@link IRegistryDriver}. When provided, the current generation is resolved via the registry's
+     * authoritative `currentGen` (one cheap strong read) instead of a `list` scan of every generation — the
+     * retirement of that scan. When absent, the source falls back to the `list`-scan (so the
+     * in-memory / simple setups keep working with no registry). The resolved generation is cached and
+     * **re-resolved on a short TTL** ({@link currentGenTtlMs}, needs a {@link clock}) so a long-lived source
+     * observes a load's new generation within the TTL instead of pinning one generation forever.
+     */
+    readonly registry?: IRegistryDriver;
+    /**
+     * Optional {@link IKeystore} for reading **encrypted** segments. When a segment's registry record
+     * carries wrapped DEK(s), the source unwraps the DEK via this keystore and decrypts each chunk/index. Reading
+     * an encrypted segment without a keystore throws {@link KeyUnavailableError}; cleartext segments ignore it.
+     * Requires a `registry` (that's where the wrapped DEKs live).
+     */
+    readonly keystore?: IKeystore;
+    /**
+     * Enforce that every segment read is encrypted. When true, resolving a **cleartext** segment (no
+     * wrapped DEKs) throws {@link KeyUnavailableError} — a guard against silently reading a segment that should
+     * have been encrypted. Off by default (encryption is opt-in).
+     */
+    readonly requireEncryption?: boolean;
+    /**
+     * Time source for the current-generation TTL refresh — the determinism seam; `core/` never reads
+     * ambient time. Refresh needs **both** a clock and a `registry`; without either the source **pins** the
+     * first-resolved generation for its lifetime (the behaviour before the registry was wired in). The `CloudRoaring` facade passes its
+     * clock automatically, so wiring a `registry` is enough to get live cross-generation invalidation.
+     */
+    readonly clock?: Pick<Clock, 'now'>;
+    /**
+     * How long (ms) a resolved `currentGen` is trusted before the next read re-resolves it (default 2000) — the
+     * bound on read staleness after a load publishes: a reader may serve the prior generation for up to this long,
+     * then converges. Lazy (checked on read — no timer); ≤ one cheap registry read per segment per window, and a
+     * new {@link CrbmReader} is opened only when the generation actually changed. `0` (or no clock) ⇒ pin forever.
+     */
+    readonly currentGenTtlMs?: number;
+    /**
+     * Hard ceiling on how many segments' readers (each holding a fully-parsed `.crbm` index) are cached at once
+     * (default 1024) — a memory bound for a long-running server. Past it, the least-recently-used
+     * segment's reader is evicted; the next read of an evicted segment re-opens it (one cheap tail GET, since
+     * generations are immutable). Raise it for a big cache working set of small segments.
+     */
+    readonly maxOpenSegments?: number;
+    /**
+     * Aggregate byte ceiling on the parsed `.crbm` indices resident in the reader cache (default 64 MiB) — the
+     * **second half of that memory bound**. `maxOpenSegments` alone bounds by *count*, but a wide/dense segment's
+     * parsed index can be several MB, so 1024 wide indices could pin ~GBs and blow a small heap (e.g. a 128 MB
+     * Lambda) while the count is nominally "in bounds". This caps the summed {@link CrbmReader.retainedIndexBytes}
+     * across cached readers; the least-recently-used reader is evicted once the total would exceed it — whichever
+     * of the count/byte bounds binds first. Lower it for memory-tight deployments with wide segments; a single
+     * segment whose index alone exceeds the budget is still cached (it can't be shrunk) but nothing else alongside.
+     */
+    readonly maxOpenIndexBytes?: number;
+}
+export declare class CrbmStorageChunkSource implements StorageChunkSource {
+    private readonly driver;
+    /**
+     * One resolved reader per segment, re-resolved on a short TTL ({@link CrbmStorageChunkSourceOptions.currentGenTtlMs},
+     * needs a clock). Within the TTL a segment's Storage bytes are treated as an immutable snapshot; when the TTL
+     * elapses the next read cheaply re-resolves `currentGen` and, only if it advanced (a load published),
+     * opens the new generation — so a long-lived source observes new generations within the TTL rather than
+     * pinning one forever. The engine pairs this with a **generation-keyed** cache so a bump never
+     * serves a stale decoded chunk. Without a clock or a registry the source pins the first generation for its
+     * lifetime (the behaviour before the registry was wired in). A segment with no generation yet is not memoized, so it's re-checked until
+     * one exists. **Bounded** by a {@link BoundedLru} ({@link CrbmStorageChunkSourceOptions.maxOpenSegments}, default
+     * 1024): past the ceiling the least-recently-used segment's reader (and its parsed index) is evicted — the
+     * steady-state memory bound; re-opening an evicted segment is one cheap tail GET.
+     */
+    private readonly snapshots;
+    private readonly registry;
+    private readonly keystore;
+    private readonly requireEncryption;
+    private readonly readerOptions;
+    private readonly clock;
+    private readonly currentGenTtlMs;
+    constructor(driver: IStorageDriver, options?: CrbmStorageChunkSourceOptions);
+    /** The current resolved reader for a segment, refreshing on the TTL. Cheap within the TTL window. */
+    private resolvedReader;
+    /** TTL elapsed: cheaply re-resolve `currentGen`; reopen only if it actually changed, else reuse the prior reader. */
+    private computeRefreshedReader;
+    /**
+     * Memoize a reader-promise as the segment's snapshot at `now()`, forgetting it later if it resolves to
+     * null / throws (identity-guarded, so a stale cleanup never clobbers a fresher snapshot). Installing the
+     * (pending) promise **synchronously** is what lets concurrent callers coalesce onto one in-flight resolve.
+     */
+    private install;
+    private now;
+    private expired;
+    /**
+     * The segment's current generation number — the engine keys its chunk cache by this so a generation bump
+     * is observed instead of serving a stale decoded chunk. Served from the (TTL-refreshed)
+     * snapshot, so no extra backend read within the TTL window. `null` if the segment has no committed generation.
+     *
+     * Heals a swept generation exactly like {@link withFreshSnapshot} — the engine calls this **once per op**,
+     * before any chunk fetch, so an unhealed miss here fails the whole operation rather than one chunk. It is
+     * spelled out rather than delegated because this is the hottest call in the library: once per operand of
+     * every `has`/`count`/`iterate`/`intersect`, and almost always served from the cached snapshot with no
+     * backend call at all. Routing it through the generic helper cost ~115 ns/op on that path (a second async
+     * frame, a per-call closure, and an `await` on a plain number) for a race that fires only during a
+     * concurrent sweep. One retry, then propagate — same contract, same eviction rule.
+     */
+    currentGeneration(ref: SegmentRef): Promise<number | null>;
+    /**
+     * Forget everything derived from `ref`: the resolved snapshot and the open reader behind it (and with the
+     * reader, the DEK it unwrapped at open time). The next read resolves `currentGen` from the registry again.
+     *
+     * Needed because the TTL refresh only ever answers "has the pointer moved?", and every *destructive* verb —
+     * an erasure that deletes the generation holding the bit, a `dropSegment`, a crypto-shred, a retirement —
+     * leaves the pointer's answer unchanged from this source's point of view while making the snapshot wrong.
+     * Until this is called the source keeps serving that snapshot with no backend read at all, so nothing on the
+     * storage side can close the window; with `cache.genTtlMs: 0` or no clock it never closes.
+     */
+    invalidate(ref: SegmentRef): void;
+    /**
+     * Whether the segment is a registered name at all. With a registry that is exactly "a row exists" — including
+     * a row with no generation yet (minted by `setRetention` before the first load) and a `destroyed` tombstone,
+     * both of which are names somebody deliberately created. Without a registry there is no registration to
+     * consult, so the honest answer is whether the bucket holds any generation of it.
+     *
+     * Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
+     * tell apart.
+     */
+    /**
+     * `<generation>` for a registry-less source (no row, so no incarnation to confuse), or
+     * `<generation>:<row token>` with one. The token moves on every row write, so an unrelated write (a
+     * `setRetention`) costs the segment's decoded chunks once — bounded, and on an admin path. Exactness in the
+     * direction that matters: two incarnations can never share a version string.
+     */
+    currentVersion(ref: SegmentRef): Promise<string | null>;
+    /**
+     * Resolve the segment **once** and report what a pin should hold: the generation, and the version that
+     * identifies those exact bytes. `null` when the segment resolves to no generation — there is nothing to pin,
+     * and a caller must treat that as "this handle reads empty", not "pinning is unsupported here".
+     */
+    pinGeneration(ref: SegmentRef): Promise<{
+        generation: number;
+        version: string;
+    } | null>;
+    /**
+     * A reader for one **specific** generation, memoized in the same bounded LRU as the live snapshots under a
+     * generation-qualified key.
+     *
+     * Sharing the LRU is the point, not an implementation detail. A pinned reader held in a private field is
+     * outside the memory ceiling the library advertises — measured at 10.34 MiB per live pin, 32 pins holding
+     * 111.7 MiB against an 8 MiB configured bound. A pinned *generation number* has no such problem: the
+     * generation is immutable, so eviction is harmless and re-opening at the same number reproduces the same
+     * bytes. It also removes a memoized-rejection bug by construction — `install` forgets a promise that rejects,
+     * where a hand-rolled `this.reader ??= open()` cached the rejection for the life of the handle and made a pin
+     * the one read path with no resilience.
+     *
+     * **`status` is re-checked here**, on every open rather than once at pin time. A pin taken before a
+     * crypto-shred must not keep unwrapping a DEK the shred destroyed; a destroyed segment resolves no generation
+     * for anyone, pinned or not.
+     */
+    private readerAt;
+    private openAt;
+    /** The row's current key wrappings, for opening a pinned generation. */
+    private deksFor;
+    /** Read one chunk of a specific generation — the pinned read path. */
+    getChunkAt(ref: ChunkRef, generation: number): Promise<Uint8Array | null>;
+    /** Chunk keys of a specific generation — the pinned shape read. */
+    listChunkKeysAt(ref: SegmentRef, generation: number): Promise<number[]>;
+    /** Per-chunk cardinalities of a specific generation — powers a pinned `count()` with no payload reads. */
+    cardinalitiesAt(ref: SegmentRef, generation: number): Promise<ReadonlyMap<number, number> | null>;
+    /** Grounded size of a specific generation. */
+    sizeOfAt(ref: SegmentRef, generation: number): Promise<SegmentSize | null>;
+    exists(ref: SegmentRef): Promise<boolean>;
+    /**
+     * Evict a snapshot we just failed to read from — matched by its reader *promise*, so a concurrent call's
+     * fresher snapshot is never clobbered. The identity guard is the whole point: `install` replaces the entry
+     * wholesale, so comparing anything else would drop a good snapshot on the floor.
+     */
+    private dropStale;
+    private resolveLatest;
+    /** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
+    private openForTarget;
+    /**
+     * The current generation + its DEK wrappings: the registry's authoritative record (one strong read), or a
+     * `list` scan for the max generation when there's no registry (a registry-less source can only read cleartext
+     * — there's nowhere a wrapped DEK could live).
+     */
+    private resolveTarget;
+    /** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
+    private cryptoForRead;
+    getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
+    listChunkKeys(ref: SegmentRef): Promise<number[]>;
+    sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
+    cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
+    /**
+     * Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
+     * generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
+     * {@link NotFoundError}. Surfacing that as a query failure would turn a benign, recoverable race into a
+     * caller-visible error, so instead we drop the stale snapshot,
+     * re-resolve `currentGen`, and retry once — the read then serves the newer (committed, immutable) generation,
+     * a monotonic move forward — within one incarnation of the row. A name that was retired and re-created is a
+     * different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's token
+     * as its lineage rather than trusting the generation number (invariant 1).
+     *
+     * **Resolution and open are inside the retry, not before it.** Resolving `currentGen` and opening that
+     * generation's object are two backend round trips with a gap between them, so the object can be swept after
+     * the registry named it and before its tail is read — and that miss surfaces from awaiting the snapshot, not
+     * from `read`. Awaiting outside the retry left the *first* attempt unhealed on the one path GC actually
+     * races: `keep: 0`, which every id erasure passes, sweeps microseconds after the publish.
+     *
+     * A *second* NotFound is pathological (GC outrunning resolution) and propagates rather than fabricating an
+     * absent answer — never return a wrong result. `ifGone` is returned when the segment has no committed
+     * generation to serve: storage is empty, or the row was dropped or crypto-shredded, in which case reading empty
+     * is the documented outcome rather than a failure.
+     *
+     * **Bounded to exactly two resolve-and-open round trips.** That is a cost contract, not a detail: the retry
+     * now re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
+     * tight loop on a segment whose pointer names an object that is permanently absent — and an N-way
+     * `intersect` would do it N times. Gated in `tests/core/storage-source-heal-open.test.ts` by counting calls.
+     */
+    private withFreshSnapshot;
+}
+/**
+ * Write one immutable generation from in-memory bitmaps (the seed / bulk-load primitive). Chunks are
+ * sorted ascending and empty bitmaps skipped (empty chunks are never stored). Returns the driver's
+ * `{ size, sha256 }` for the written object. Pass `options.crypto` to AES-256-GCM-encrypt the generation
+ * (built from the segment's DEK, with associated data bound to `(segment, generation)`).
+ */
+export declare function writeCrbmGeneration(driver: IStorageDriver, key: GenKey, chunks: Iterable<{
+    chunkKey: number;
+    bitmap: CodecBitmap;
+}>, options?: {
+    crypto?: CrbmCrypto;
+    clock?: Yielder;
+}): Promise<{
+    size: number;
+    sha256: string;
+}>;
+/** What {@link writeCrbmGenerationStream} wrote: the driver's `{ size, sha256 }` + a tally of the generation. */
+export interface StreamWriteResult {
+    readonly size: number;
+    readonly sha256: string;
+    /** Non-empty chunk keys written, in ascending order (for the post-write verify, since the stream is consumed). */
+    readonly chunkKeys: number[];
+    /** Total ids written. */
+    readonly cardinality: number;
+}
+/**
+ * Streaming variant of {@link writeCrbmGeneration} for a **constant-memory** rewrite (the erasure rewrite, a
+ * sorted load): consumes an **already-ascending** async stream of `{ chunkKey, bitmap }`, feeding each to the
+ * codec and freeing it, instead of materializing the whole generation. Paired with a streaming storage sink (S3
+ * multipart / LocalFs temp file), peak memory is ~one chunk + one part. Returns a tally so the caller can verify
+ * the re-opened object without re-iterating the (now-consumed) stream. Input **must** be ascending by `chunkKey`
+ * — the codec rejects an out-of-order chunk; empty bitmaps are skipped.
+ */
+export declare function writeCrbmGenerationStream(driver: IStorageDriver, key: GenKey, chunks: AsyncIterable<{
+    chunkKey: number;
+    bitmap: CodecBitmap;
+}>, options?: {
+    crypto?: CrbmCrypto;
+    clock?: Yielder;
+}): Promise<StreamWriteResult>;
+/**
+ * Point a segment's registry `currentGen` at `key.generation` — the publish step that makes a freshly-written
+ * generation the authoritative latest (so registry-aware readers see it). **Forward-only and idempotent:**
+ * if the registry has no row it creates one; if it's already at/ahead of `key.generation` it's a no-op (an
+ * out-of-order/duplicate publish never regresses the pointer); otherwise it advances via compare-and-swap,
+ * retrying a few times under contention. Separated from the Storage write so callers can publish atomically
+ * after the immutable object is durable (write-then-publish).
+ *
+ * **Throws** rather than returning, in two cases, because each would otherwise create a state no reader can
+ * recover from: a `destroyed` (crypto-shredded) row, where advancing the pointer would name an object encrypted
+ * under a shredded key; and an advance carrying **new** `wrappedDeks`, which would either strand that key or make
+ * the row advertise encryption over cleartext generations. Contention that never clears throws
+ * {@link WriteConflictError} after the retries.
+ *
+ * **`expectFrom` turns forward-only into read-modify-write.** Forward-only is the right rule for a writer whose
+ * content does not depend on what was current — a load computes its ids upstream, so publishing over a newer
+ * generation loses nothing the loader knew about. It is the WRONG rule for a writer that *derived* its content
+ * from a particular generation: the erasure rewrite streams generation `from` and clears one bit, so publishing
+ * it over a newer generation silently discards whatever that generation added. Passing `expectFrom` makes the
+ * publish land only while the pointer is still exactly there, and return `false` otherwise, so the caller can
+ * report `superseded` and re-derive. The compare-and-swap itself carries the token read in the same iteration,
+ * so the check and the write see the same row.
+ *
+ * **`expectToken` fences the LINEAGE, and a derived writer needs both.** A generation number identifies a
+ * generation only within one incarnation of a name: `nextGeneration` restarts at `0` once the row is purged and
+ * the bucket empty, so a name that is retired and re-created has a *different* segment wearing the *same*
+ * `currentGen`. `expectFrom` alone matched it — and an erasure rewrite then published one incarnation's content
+ * over another's, deleted the live objects with its `keep: 0` collection, and returned `erased: true`. The row's
+ * OCC token is the identity that survives this: {@link IRegistryDriver.delete} tombstones rather than unlinks,
+ * so "a later `create` still gets a fresh, greater token" and no token is ever reused across incarnations.
+ *
+ * Pass the token read alongside `expectFrom` and the publish lands only on the same row it was derived from.
+ * The check is deliberately **conservative**: the token also changes on writes that are not supersessions at
+ * all (a `setRetention`, a due-index reindex), so such a write makes a derived publish report `superseded` and
+ * the caller re-derive. That costs a re-run on a rare, unrelated write; the alternative costs a segment.
+ */
+export declare function publishGeneration(registry: IRegistryDriver, key: GenKey, options?: {
+    wrappedDeks?: readonly WrappedDek[];
+    expectFrom?: number;
+    expectToken?: Token;
+    /**
+     * Publish only while the segment still has **no registry row**.
+     *
+     * The fence for a caller whose decision rests on the row being ABSENT. `expectFrom` and `expectToken`
+     * cannot express it: both compare against a value read from a row, so when there was no row there is
+     * nothing to compare and both are simply omitted — leaving the publish a bare forward-only advance that
+     * happily lands over whatever appeared in the meantime.
+     *
+     * That gap was a silent wipe, not a theoretical one. A guarded load into a segment that did not exist yet
+     * read "no row", so its empty/`minRetained` bounds had nothing to judge and passed vacuously; a
+     * concurrent writer then created the row and published a thousand ids; and the guarded load published an
+     * EMPTY generation over them, reporting `published: true` with no reason. Reproduced through both
+     * `loadSegment` and the `*Into` verbs, which hit it far more often because materialising into a
+     * destination that does not exist yet is the ordinary first run of a pipeline.
+     */
+    expectAbsent?: boolean;
+}): Promise<boolean>;
+/** What a bulk-load wrote — the driver's `{ size, sha256 }` plus a summary of the built generation. */
+export interface BulkLoadResult {
+    /** Bytes written to the Storage object. */
+    readonly size: number;
+    /** The driver's content hash of the object. */
+    readonly sha256: string;
+    /** Distinct non-empty chunks written (≤ 65536). */
+    readonly chunkCount: number;
+    /**
+     * The DEK freshly minted for this generation, wrapped — present only when the segment is encrypted AND this
+     * call minted one (an existing segment reuses its key, which is already on the row).
+     *
+     * Returned so a caller that deliberately does **not** pass `registry` here — writing now and publishing later,
+     * which is how a load applies a guard while the old generation is still authoritative — can carry the key onto
+     * its own `publishGeneration`. Without it that publish stores no DEK and the generation it makes current is
+     * unreadable: the bytes are encrypted with a key nothing recorded.
+     */
+    readonly wrappedDeks?: readonly WrappedDek[];
+    /** Total distinct ids in the generation (post-dedup). */
+    readonly cardinality: number;
+    /**
+     * Whether this generation is now the segment's **current** one — `undefined` when no `registry` was wired, so
+     * there is no pointer and no publish step to report on.
+     *
+     * `false` means the object is durable but a **concurrent writer published a higher generation first**, so this
+     * one is an orphan that no reader will ever resolve. The write succeeded and the *load* did not: whatever this
+     * call was asked to make the segment contain, the segment does not contain. That is not a detail a caller can
+     * be left to infer — a `*Into` materialisation used to report the generation it wrote as "the destination's
+     * new current generation" on exactly this path, which is a plain untruth — so the flag is on the result and the
+     * verbs that promise a published generation check it.
+     */
+    readonly becameCurrent?: boolean;
+}
+/**
+ * **Bulk-load** a whole immutable generation from a flat id source — the batch "seed/rebuild" entry point.
+ * Unlike {@link writeCrbmGeneration} (which takes pre-grouped bitmaps), this
+ * consumes an arbitrary, **unsorted** stream of u32 ids (sync or async) and routes each into its chunk's
+ * Roaring bitmap as it goes. The input is consumed lazily and ids dedup on insert, so memory is bounded by
+ * the **built generation** — the in-memory Roaring representation of the *distinct* set (≤ one bitmap per
+ * non-empty chunk) — not by the input length: you can stream a billion duplicate-heavy ids and hold only the
+ * distinct result. Note this is `O(distinct set)`, **not** window-bounded like {@link SegmentEngine.intersect}:
+ * bulk-load holds the whole generation in RAM, which suits batch seed/rebuild jobs but not an
+ * unbounded-cardinality stream (a segment larger than RAM needs external-merge / pre-sorted input — later phase).
+ *
+ * Each id must be an integer in `[0, 2³²)` ({@link splitId} throws {@link ValidationError} otherwise) — the
+ * source is consumed lazily, so a bad id aborts mid-stream without writing a partial object (the driver
+ * commits only after the callback resolves). An empty source writes a valid empty generation.
+ *
+ * Writes a fresh full snapshot of a segment. The caller picks the generation number in `key` — `nextGeneration`
+ * computes the right one from the registry and the bucket — and re-using an existing generation throws
+ * {@link WriteConflictError} (write-once). Without a registry a `StorageChunkSource` serves the **highest**
+ * generation present, so a too-high number silently shadows real data; with one, `publishGeneration` decides.
+ */
+export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenKey, ids: Iterable<number> | AsyncIterable<number>, options?: {
+    registry?: IRegistryDriver;
+    keystore?: IKeystore;
+    requireEncryption?: boolean;
+    audit?: IAuditSink;
+    /** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
+    codec?: CodecInterface;
+    /**
+     * Injected clock. Supplying one makes bulk-load **cooperative**: it yields the event loop periodically so a
+     * long load does not stall everything else on the process. Without it the load still completes, just
+     * without yielding — which is the pre-existing behaviour, kept so this is purely additive.
+     *
+     * `@cloudbitmaps/roaring` supplies a real clock by default, so flavor users get cooperative behaviour with
+     * no wiring. `core/` cannot default it: it is timer-free by lint, which is exactly why waiting goes through
+     * this seam rather than `setTimeout`.
+     */
+    clock?: Clock;
+    /**
+     * Write the object but do **not** advance the pointer (default: publish when a `registry` is wired).
+     *
+     * For a caller that has to inspect what it wrote before deciding whether it should become current — a
+     * guarded {@link loadSegment} is the one in-tree case — because the check is only meaningful while the old
+     * generation is still authoritative. The `registry` is still required for an encrypted segment and still
+     * consulted: it is where an existing segment's DEK lives, and reusing that key is not optional. The wrapped
+     * DEK comes back on the result so the deferred publish can store it.
+     *
+     * A caller that defers the publish owns what it wrote: an unpublished object sits ABOVE `currentGen`, where
+     * generation collection deliberately never looks, so nothing else will ever reclaim it.
+     */
+    publish?: boolean;
+}): Promise<BulkLoadResult>;
+/**
+ * Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
+ * The reader the write paths use to re-read what they wrote, and the erasure rewrite uses to stream the old
+ * generation; the engine's read path goes through {@link CrbmStorageChunkSource} instead, which caches these.
+ */
+export declare function openGenerationReader(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, options?: Omit<CrbmReaderOptions, 'crypto'>): Promise<CrbmReader>;
+/**
+ * Re-open a freshly written generation and assert it round-trips exactly what was streamed into it: the same
+ * per-chunk key set *and* the same total cardinality (on top of the codec's own per-chunk CRC + footer checks).
+ * `expected` is the streaming writer's tally (the stream is consumed, so it can't be re-iterated) — the key-set
+ * comparison catches a dropped/extra chunk that a cardinality-only check could miss when two errors cancel out.
+ * Throws {@link IntegrityError}: the object is on disk but must not be published.
+ */
+export declare function verifyGeneration(storage: IStorageDriver, key: GenKey, expected: {
+    readonly chunkKeys: readonly number[];
+    readonly cardinality: number;
+}, crypto: CrbmCrypto | undefined): Promise<void>;

package/dist/core/crypto.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Encryption seams — the **pure** interfaces `core/` depends on so the codec can encrypt/decrypt without
+ * importing `node:crypto` (lint-banned here, like `Clock`/`Rng`/drivers). Concrete AES-256-GCM lives outside
+ * `core/` (`src/drivers/crypto.ts`) and is injected. The envelope scheme: a per-segment DEK wrapped under an
+ * operator-held KEK, so discarding the DEK crypto-shreds the segment. Per-chunk AEAD over the `.crbm` payloads + index; a per-segment **DEK** wrapped (envelope
+ * encryption) under one or more **KEK**s by an {@link IKeystore}; the wrapped DEK(s) live in the registry, so
+ * crypto-shred = delete them.
+ */
+/** One AEAD result: a fresh per-message nonce, the ciphertext, and the authentication tag. */
+export interface AeadSealed {
+    readonly nonce: Uint8Array;
+    readonly ciphertext: Uint8Array;
+    readonly tag: Uint8Array;
+}
+/**
+ * Authenticated encryption bound to a single key (a segment's DEK). `aad` (associated data) is authenticated
+ * but not encrypted — the codec passes each chunk's `(namespace, segment, generation, chunkKey)` identity
+ * (the library derives it; an `Aead` implementation receives it), so a chunk's ciphertext can't be silently
+ * relocated to another segment/generation/chunk or
+ * the index, even by someone who can rewrite Storage objects. The seam is symmetric and self-framing-agnostic:
+ * the caller decides where the nonce/tag go (inline with a chunk, or in the footer for the index).
+ */
+export interface Aead {
+    /** Encrypt `plaintext`, authenticating `aad`. Returns a fresh nonce + ciphertext + tag. */
+    seal(plaintext: Uint8Array, aad: Uint8Array): AeadSealed;
+    /**
+     * Decrypt + verify. Throws {@link IntegrityError} on **any** mismatch (wrong key, tampered ciphertext/tag,
+     * or `aad` that doesn't match what was sealed) — never returns wrong-but-plausible plaintext.
+     */
+    open(sealed: AeadSealed, aad: Uint8Array): Uint8Array;
+}
+/**
+ * A DEK wrapped under one KEK. A segment stores a **list** of these (one per KEK it was wrapped under — e.g.
+ * an active KEK plus an offline recovery KEK), so losing one KEK isn't fatal and KEK rotation needs no
+ * data re-encryption. `keyId` names which KEK; `wrapped` is the opaque wrapped-key blob (base64). Carries no
+ * plaintext key material — safe to store in the registry next to the (encrypted) data.
+ */
+export interface WrappedDek {
+    readonly keyId: string;
+    readonly wrapped: string;
+}
+/**
+ * Envelope key management — the injected seam between the codec and however the operator holds key material
+ * (in-process BYOK by default; KMS/Vault as optional adapters later). Async so a remote keystore (KMS) fits
+ * the same interface. Never exposes raw key bytes to `core/`: it returns an {@link Aead} bound to the DEK.
+ */
+export interface IKeystore {
+    /**
+     * Mint a fresh per-segment DEK, wrapped under the active KEK (and any recovery KEK). Returns the wrapping(s)
+     * to persist in the registry plus an {@link Aead} to encrypt this segment's chunks with.
+     */
+    createDek(): Promise<{
+        wrapped: WrappedDek[];
+        aead: Aead;
+    }>;
+    /**
+     * Reconstruct the {@link Aead} from a segment's stored wrappings by unwrapping with any KEK currently held.
+     * Throws {@link KeyUnavailableError} if it holds none of the referenced KEKs (lost key, or a shredded
+     * segment whose wrappings were deleted).
+     */
+    openDek(wrapped: readonly WrappedDek[]): Promise<Aead>;
+}
+/**
+ * What the encrypting codec needs: the {@link Aead} for this segment's DEK plus a way to build the AAD for a
+ * given scope (`'index'` or a chunkKey). Built per (segment, generation) by the storage-source bridge so the
+ * codec itself stays segment-agnostic.
+ */
+export interface CrbmCrypto {
+    readonly aead: Aead;
+    aadFor(scope: number | 'index'): Uint8Array;
+}
+/**
+ * Build the AEAD associated-data binding a chunk (or the index) to its exact location:
+ * `v1 ‖ len(namespace) ‖ namespace ‖ len(segment) ‖ segment ‖ generation ‖ scope ‖ chunkKey`. Length-prefixed
+ * so distinct `(namespace, segment)` pairs can never collide. Pure (no crypto) — just the authenticated label.
+ */
+export declare function aadFor(ref: {
+    readonly namespace?: string;
+    readonly segment: string;
+}, generation: number, scope: number | 'index'): Uint8Array;

package/dist/core/determinism.d.ts

@@ -0,0 +1,42 @@
+/**
+ * The determinism seam.
+ *
+ * `core/` is a pure function of its inputs + injected dependencies — it never reaches for
+ * ambient time or randomness. Production wiring supplies real implementations; the deterministic
+ * tests supply controlled, seeded ones, so any run can be replayed exactly from a seed.
+ *
+ * These are interfaces only: the seam the whole library depends on for reproducibility. Every module that
+ * needs time, randomness or a yield takes one of these rather than reaching for a global.
+ */
+/** Injected time source. `core/` must never call `Date.now()` directly. */
+export interface Clock {
+    /** Milliseconds since the Unix epoch. */
+    now(): number;
+    /**
+     * Resolve after at least `ms` have elapsed. The retry/backoff path's only way to wait — `core/` must
+     * never reach for `setTimeout` directly (it's timer-free, lint-enforced). Production wiring supplies a
+     * `setTimeout`-backed implementation; a test supplies a virtual-time one so a backoff schedule is
+     * replayable. `ms <= 0` resolves on a microtask without scheduling a timer.
+     */
+    sleep(ms: number): Promise<void>;
+    /**
+     * Hand the event loop back once, without asking for any elapsed time.
+     *
+     * This is **not** `sleep(0)`, and the difference is the reason the member exists. `sleep(0)` resolves on a
+     * microtask, and microtasks drain before the loop advances a phase — so awaiting it inside a CPU-bound loop
+     * yields to nothing at all, and a co-resident server stays blocked for the loop's full duration. A real yield
+     * needs a macrotask. `sleep(1)` is one, but it buys the relief at ~1 ms of dead wall-clock per yield.
+     *
+     * **Optional**, so every `Clock` written before this member still satisfies the interface. Callers must
+     * therefore degrade rather than assume: `clock.yieldNow?.() ?? clock.sleep(1)` — correct on any clock, cheap
+     * on one that implements this. `core/` cannot supply it itself (it is timer-free, lint-enforced); production
+     * wiring backs it with `setImmediate`, and a test can make it a no-op so virtual time is not perturbed
+     * by what is purely a scheduling courtesy.
+     */
+    yieldNow?(): Promise<void>;
+}
+/** Injected randomness. `core/` must never call `Math.random()` directly. Seedable for simulation. */
+export interface Rng {
+    /** A float in the half-open interval [0, 1). */
+    next(): number;
+}

package/dist/core/discover.d.ts

@@ -0,0 +1,75 @@
+/**
+ * `segmentExists` and `listSegments` — ask the registry what is there.
+ *
+ * The registry has always known which segments exist; it is what `checkConsistency`, the retention sweep and
+ * `exportSegments` each enumerate. It was simply never exposed, so a user asking "do I already have this?" had
+ * to infer it: `count()` returns `0` for a segment that was never loaded AND for one loaded with no ids, and
+ * `generations()` costs a bucket listing to answer a question the pointer alone settles.
+ *
+ * Inferring it is the expensive kind of wrong. The alternative a user reaches for is keeping their own list of
+ * segment names alongside the store — a second source of truth, updated by hand, drifting from the first the
+ * moment a load fails halfway. The registry is already the list; these two functions read it.
+ */
+import type { IRegistryDriver, RegistryRecord, SegmentRef } from './ports.js';
+/** One segment the registry knows about. */
+export interface SegmentInfo extends SegmentRef {
+    /**
+     * The generation the **pointer names** — not what a read would resolve. On a `destroyed` tombstone this is
+     * still a number while reads answer empty, so read it together with {@link SegmentInfo.status} rather than
+     * alone. `null` means the row exists with no Storage generation yet.
+     */
+    readonly currentGen: number | null;
+    /**
+     * `destroyed` is a tombstone: the row stays so sweeps can still find it, and the generations are gone — on an
+     * encrypted segment the DEK wrappings went with them, which is what makes the bytes unrecoverable rather than
+     * merely deleted.
+     */
+    readonly status: RegistryRecord['status'];
+}
+/**
+ * Whether this segment's pointer resolves a generation — the question behind "does it already exist?".
+ *
+ * One registry point read, on every backend. Deliberately narrower than "is there a row": it is **false** for a
+ * row that exists with `currentGen: null` (a retention policy recorded ahead of the first load) and for a
+ * `destroyed` tombstone, because a read answers empty in both. The predicate mirrors generation resolution
+ * clause for clause, which is what keeps the two from disagreeing.
+ *
+ * It is NOT the same question as `count() > 0`. A segment loaded with no ids exists and counts zero — the
+ * distinction between "never loaded" and "loaded, and genuinely empty" is the one `count()` cannot make and
+ * the main reason this function is worth having.
+ *
+ * Two states answer `true` where a read still gives you nothing, and neither is this call's job to detect: a
+ * **torn restore** (`missing-storage-generation` — a live pointer whose object was deleted) makes reads *throw*
+ * `NotFoundError` rather than answer empty, and `runConsistencyCheck` is the call that looks for it; and a
+ * handle carrying an expired `expiresAt` reads as empty by a rule that lives on the handle, not on the row.
+ *
+ * Not a lock. A segment can be created or dropped between this call and whatever you do next; if the answer
+ * has to hold, use the fences that exist for it — `load`'s guard, or `expectFrom`/`expectToken` on a publish.
+ */
+export declare function segmentExists(ref: SegmentRef, registry: IRegistryDriver): Promise<boolean>;
+/**
+ * Every segment the registry holds, streamed — optionally scoped to one namespace.
+ *
+ * **This is an admin/discovery call, not a request-path one.** It is the registry's own enumeration — a paged
+ * LIST over the `registry/` prefix — so cost grows with the fleet, not with what you are looking for. The
+ * fleet-wide sweeps that *drain* it bound it (`maxScanSegments`); this one streams, so the bound is yours.
+ *
+ * **Scoping to a namespace narrows the LIST prefix**, so it really is the difference between reading one
+ * tenant and reading all of them.
+ *
+ * Streams, so a large fleet need not be held at once, and stopping the iteration stops the scan — **with one
+ * exception that matters**: a driver that buffers its enumeration defeats both properties, and
+ * `RetryingRegistryDriver` does exactly that (it must, to retry a `list` as a unit). Wrapped in it, the whole
+ * scan is paid for and resident before the first row reaches you. On the four native drivers the guarantee
+ * holds; on S3 the granularity is a page, so one LIST page and its in-flight row reads complete regardless.
+ *
+ * Yields `destroyed` tombstones, and rows whose `currentGen` is `null`, because hiding either would make this
+ * disagree with the registry it reports on — a filtered enumeration that looks complete is how a retention
+ * sweep ends up permanently skipping rows nobody can see. Filter on `status`/`currentGen` yourself, or use
+ * {@link segmentExists} for the "does the pointer resolve" question. Internal bookkeeping rows are the one
+ * exclusion, and only on an unscoped scan: they are not segments, and a caller naming their namespace still
+ * sees them.
+ */
+export declare function listSegments(registry: IRegistryDriver, options?: {
+    namespace?: string;
+}): AsyncIterable<SegmentInfo>;