@cloudbitmaps/core 0.9.0 → 0.11.0

package/dist/core/crbm/writer.d.ts

@@ -0,0 +1,39 @@
+import type { CrbmCrypto } from '../crypto.js';
+import type { BlobSink } from '../blob.js';
+export interface CrbmWriterOptions {
+    /** Self-describing generation number (also encoded in the object key). */
+    readonly generation: number;
+    /**
+     * Which codec produced the chunk payloads. Defaults to {@link PAYLOAD_CODEC_ROARING_PORTABLE}.
+     *
+     * A codec id rather than a roaring-specific one, because `.crbm` is a shared container and a second codec is
+     * expected before the format freezes.
+     */
+    readonly payloadCodecId?: number;
+    readonly elementWidth?: number;
+    /** When set, payloads + index are AES-256-GCM-encrypted, bound to `(segment, generation, scope)` via AAD. */
+    readonly crypto?: CrbmCrypto;
+}
+export declare class CrbmWriter {
+    private readonly sink;
+    private readonly options;
+    private readonly entries;
+    private offset;
+    private lastKey;
+    private totalCardinality;
+    private preambleWritten;
+    private finished;
+    constructor(sink: BlobSink, options: CrbmWriterOptions);
+    /**
+     * Append one chunk's payload (the roaring-serialized bytes) with its cardinality. `chunkKey` must be
+     * strictly greater than the previous one (ascending, no duplicates); `cardinality` is `[1, 65536]`.
+     */
+    addChunk(chunkKey: number, payload: Uint8Array, cardinality: number): Promise<void>;
+    /** Seal one chunk payload into the on-disk frame `nonce ‖ ciphertext ‖ tag`, bound to its chunkKey. */
+    private sealChunk;
+    /** Write the index region + the fixed footer. After this the object is complete and immutable. */
+    finish(): Promise<void>;
+    private ensurePreamble;
+    private encodeIndex;
+    private buildFooter;
+}

package/dist/core/crbm-storage-source.d.ts

@@ -0,0 +1,522 @@
+import { type IAuditSink } from './audit.js';
+import type { Yielder } from './cooperative.js';
+import type { Clock } from './determinism.js';
+import type { CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
+import type { ChunkRef, StorageChunkSource, GenKey, IStorageDriver, IRegistryDriver, SegmentRef, SegmentSize, Token } from './ports.js';
+import { CrbmReader } from './crbm/reader.js';
+import type { CrbmReaderOptions } from './crbm/reader.js';
+import type { CodecBitmap, CodecInterface } from './codec.js';
+export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
+    /**
+     * Optional {@link IRegistryDriver}. When provided, the current generation is resolved via the registry's
+     * authoritative `currentGen` (one cheap strong read) instead of a `list` scan of every generation — the
+     * retirement of that scan. When absent, the source falls back to the `list`-scan (so the
+     * in-memory / simple setups keep working with no registry). The resolved generation is cached and
+     * **re-resolved on a short TTL** ({@link currentGenTtlMs}, needs a {@link clock}) so a long-lived source
+     * observes another process's load within the TTL, not only when something else happens to re-resolve it.
+     */
+    readonly registry?: IRegistryDriver;
+    /**
+     * Optional {@link IKeystore} for reading **encrypted** segments. When a segment's registry record
+     * carries wrapped DEK(s), the source unwraps the DEK via this keystore and decrypts each chunk/index. Reading
+     * an encrypted segment without a keystore throws {@link KeyUnavailableError}; cleartext segments ignore it.
+     * Requires a `registry` (that's where the wrapped DEKs live).
+     */
+    readonly keystore?: IKeystore;
+    /**
+     * Enforce that every segment read is encrypted. When true, resolving a **cleartext** segment (no
+     * wrapped DEKs) throws {@link KeyUnavailableError} — a guard against silently reading a segment that should
+     * have been encrypted. Off by default (encryption is opt-in).
+     */
+    readonly requireEncryption?: boolean;
+    /**
+     * Time source for the current-generation TTL refresh — the determinism seam; `core/` never reads
+     * ambient time. Refresh needs **both** a clock and a `registry`. Without either there is no timed refresh, and
+     * that is the only difference: a segment is still re-resolved when its reader is evicted, when a fetch finds its
+     * generation swept, and on {@link CrbmStorageChunkSource.invalidate}. What goes is the bound on how long another
+     * process's publish takes to arrive. The `CloudRoaring` facade passes its clock automatically, so wiring a
+     * `registry` is enough to get the bounded refresh.
+     */
+    readonly clock?: Pick<Clock, 'now'>;
+    /**
+     * How long (ms) a resolved `currentGen` is trusted before the next read re-resolves it (default 2000) — the
+     * bound on read staleness after a load publishes: a reader may serve the prior generation for up to this long,
+     * then converges. Lazy (checked on read — no timer); ≤ one cheap registry read per segment per window, and a
+     * new {@link CrbmReader} is opened only when the generation actually changed. `0` turns the refresh off, as a
+     * missing clock or registry does: {@link clock} says what that changes, and what it does not.
+     */
+    readonly currentGenTtlMs?: number;
+    /**
+     * Hard ceiling on how many segments' readers (each holding a fully-parsed `.crbm` index) are cached at once
+     * (default 1024) — a memory bound for a long-running server. Past it, the least-recently-used
+     * segment's reader is evicted; the next read of an evicted segment re-opens it (one cheap tail GET, since
+     * generations are immutable). Raise it for a big cache working set of small segments.
+     */
+    readonly maxOpenSegments?: number;
+    /**
+     * Aggregate byte ceiling on the parsed `.crbm` indices resident in the reader cache (default 64 MiB) — the
+     * **second half of that memory bound**. `maxOpenSegments` alone bounds by *count*, but a wide/dense segment's
+     * parsed index can be several MB, so 1024 wide indices could pin ~GBs and blow a small heap (e.g. a 128 MB
+     * Lambda) while the count is nominally "in bounds". This caps the summed {@link CrbmReader.retainedIndexBytes}
+     * across cached readers; the least-recently-used reader is evicted once the total would exceed it — whichever
+     * of the count/byte bounds binds first. Lower it for memory-tight deployments with wide segments; a single
+     * segment whose index alone exceeds the budget is still cached (it can't be shrunk) but nothing else alongside.
+     */
+    readonly maxOpenIndexBytes?: number;
+}
+/**
+ * What a pin holds of the object it pinned: the version its cache entries are keyed by, and the object's
+ * fingerprint, which a pinned read checks each time it opens the generation. `pinGeneration` always records one.
+ * A pin built by hand without one is not checked: it reads whatever object is under its generation's key.
+ */
+export interface PinnedObject {
+    readonly version: string;
+    readonly fingerprint?: string;
+}
+export declare class CrbmStorageChunkSource implements StorageChunkSource {
+    private readonly driver;
+    /**
+     * One resolved reader per segment, re-resolved on a short TTL ({@link CrbmStorageChunkSourceOptions.currentGenTtlMs},
+     * needs a clock). Within the TTL a segment's Storage bytes are treated as an immutable snapshot; when the TTL
+     * elapses the next read cheaply re-resolves `currentGen` and, only if it advanced (a load published),
+     * opens the new generation — so a long-lived source observes new generations within the TTL. The engine pairs
+     * this with a **generation-keyed** cache so a bump never serves a stale decoded chunk. Without a clock or a
+     * registry there is no timed refresh, and a segment's reader is replaced only when it is evicted, when a read
+     * has to fetch from a generation a sweep deleted, or when {@link invalidate} drops it. A segment with no generation yet is not memoized, so
+     * it's re-checked until one exists. **Bounded** by a {@link BoundedLru} ({@link CrbmStorageChunkSourceOptions.maxOpenSegments}, default
+     * 1024): past the ceiling the least-recently-used segment's reader (and its parsed index) is evicted — the
+     * steady-state memory bound; re-opening an evicted segment is one cheap tail GET.
+     */
+    private readonly snapshots;
+    /**
+     * Pins ({@link heldKey}) whose object has been found replaced: another object is under its generation's key. Kept,
+     * and bounded as the readers are, so that a pin's later chunk reads and any reopen fail at once rather than pay for
+     * the check again. A replacement can be undone, by a restore that puts the pinned object back, so
+     * {@link invalidate} forgets the segment's, and a pin of the same version that opens the object forgets the one
+     * against it. An object found gone is not kept: that is the open's own `NotFoundError`, and a 404 can pass. A
+     * reader still memoised keeps answering from the index it holds, so the pin's `count()` does, until a later pin of
+     * that version opens the object now under the key, which takes its place.
+     */
+    private readonly replacedPins;
+    /**
+     * A check of a pinned object still under way, by {@link heldKey}, so the reads that ask at once share one footer
+     * read. One forgotten before it ends, by {@link invalidate} or a pin, keeps nothing it finds.
+     */
+    private readonly checking;
+    /**
+     * How many times {@link invalidate} has run, so a read that waited can tell whether the store was told something
+     * meanwhile. An eviction from the reader cache is not an invalidation, and what a read found stands through one. It
+     * counts every segment's invalidations: a read that waited through another segment's only costs a later read the
+     * check again.
+     */
+    private invalidations;
+    private readonly registry;
+    private readonly keystore;
+    private readonly requireEncryption;
+    private readonly readerOptions;
+    private readonly clock;
+    private readonly currentGenTtlMs;
+    constructor(driver: IStorageDriver, options?: CrbmStorageChunkSourceOptions);
+    /** The current resolved reader for a segment, refreshing on the TTL. Cheap within the TTL window. */
+    private resolvedReader;
+    /**
+     * A refresh failed transiently and the prior reader keeps serving: make this snapshot lapse after
+     * {@link REFRESH_RETRY_MS} (never longer than the TTL) rather than after a whole TTL, so the next read after
+     * that asks the registry again. It costs no read of its own; it only changes when the next refresh happens.
+     */
+    private retrySoon;
+    /** TTL elapsed: cheaply re-resolve `currentGen`; reopen only if it actually changed, else reuse the prior reader. */
+    private computeRefreshedReader;
+    /**
+     * Memoize a reader-promise as the segment's snapshot at `now()`, forgetting it later if it resolves to
+     * null / throws (identity-guarded, so a stale cleanup never clobbers a fresher snapshot). Installing the
+     * (pending) promise **synchronously** is what lets concurrent callers coalesce onto one in-flight resolve.
+     */
+    private install;
+    private now;
+    /**
+     * How often this source re-reads a segment's pointer while the segment is being read, in ms: its TTL, or 0 when
+     * it has no timed refresh (no clock, no registry, or a TTL of 0). The grounded cost report prices the refresh at
+     * this.
+     */
+    get pointerRefreshMs(): number;
+    private expired;
+    /**
+     * The segment's current generation number — the engine keys its chunk cache by this so a generation bump
+     * is observed instead of serving a stale decoded chunk. Served from the (TTL-refreshed)
+     * snapshot, so no extra backend read within the TTL window. `null` if the segment has no committed generation.
+     *
+     * Heals a swept generation exactly like {@link withFreshSnapshot}: a caller that resolves the generation once
+     * per op, before any chunk fetch, would otherwise fail the whole operation rather than one chunk. The engine
+     * keys by {@link currentVersion} instead, which heals the same way, and falls back to this for a source that
+     * cannot report a version. It is spelled out rather than delegated because a lookup like this runs once per
+     * operand of every `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with
+     * no backend call at all. Routing it through the generic helper cost ~115 ns/op on that path (a second async
+     * frame, a per-call closure, and an `await` on a plain number) for a race that fires only during a
+     * concurrent sweep. One retry, then propagate — same contract, same eviction rule.
+     */
+    currentGeneration(ref: SegmentRef): Promise<number | null>;
+    /**
+     * Forget everything derived from `ref`: the resolved snapshot and the open reader behind it (and with the
+     * reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
+     * resolves the segment again.
+     *
+     * Needed because the TTL refresh only ever answers "has the pointer moved?", and every *destructive* verb —
+     * an erasure that deletes the generation holding the bit, a `dropSegment`, a crypto-shred, a retirement —
+     * leaves the pointer's answer unchanged from this source's point of view while making the snapshot wrong.
+     * Until this is called the source keeps serving that snapshot with no backend read at all, so nothing on the
+     * storage side can close the window; with no registry, no clock or `cache.genTtlMs: 0`, nothing bounds it at all.
+     */
+    invalidate(ref: SegmentRef): void;
+    /**
+     * `<generation>` for a registry-less source, or `<generation>:<row token>` with one. The token moves on every row
+     * write, so an unrelated write (a `setRetention`) costs the segment's decoded chunks once — bounded, and on an
+     * admin path. Exactness in the direction that matters: with a registry, two incarnations can never share a version
+     * string. Without one they can: a name purged and loaded again out of band restarts at the same bare number, and
+     * a live read that reopens it after an eviction reads the new object's chunks beside the old one's cached ones. A
+     * store with no registry cannot write, so that takes a change made from outside it; a pin is kept apart from it by
+     * the fingerprint it records.
+     *
+     * This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its
+     * chunk cache, so it heals a swept generation exactly as {@link currentGeneration} does: unhealed, a cold read
+     * racing a publish and a `keep: 0` sweep failed the whole operation with `NotFoundError`, where the same race
+     * on a chunk fetch heals. Spelled out rather than delegated for the reason `currentGeneration` gives: it is
+     * almost always served from the cached snapshot with no backend call at all. One retry, then propagate.
+     */
+    currentVersion(ref: SegmentRef): Promise<string | null>;
+    /**
+     * Resolve the segment **once** and report what a pin should hold: the generation, and the version that
+     * identifies those exact bytes. `null` when the segment resolves to no generation — there is nothing to pin,
+     * and a caller must treat that as "this handle reads empty", not "pinning is unsupported here".
+     *
+     * Heals a generation swept between the registry read and the open, as {@link currentVersion} does: a publish
+     * and a `keep: 0` sweep can land in that gap, and nothing has been handed out yet, so the generation current
+     * once the swept one is gone is exactly what a pin taken now should hold. One retry, then propagate.
+     */
+    pinGeneration(ref: SegmentRef): Promise<({
+        generation: number;
+    } & Required<PinnedObject>) | null>;
+    private pinOnce;
+    /**
+     * Where a pinned reader of one object is memoised. With a registry that is its version, which names the object:
+     * a name purged and loaded again gets a new row, and with it a new token. Without one the version is the bare
+     * generation number, which two incarnations share, so the object's fingerprint is added to tell them apart.
+     */
+    private pinnedKey;
+    /**
+     * One pin's hold on one segment: its version, and the fingerprint of the object it pinned. A check of that object
+     * is shared under it, and what the check finds is kept under it, with a registry or without one: pins of one
+     * version can hold different objects, since a pin built by hand can name any, and an object replaced from outside
+     * the store leaves the row, and so the version, as it was.
+     */
+    private heldKey;
+    /**
+     * A reader for one **specific** generation, memoized in the same bounded LRU as the live snapshots under a
+     * generation-qualified key.
+     *
+     * Sharing the LRU is the point, not an implementation detail. A pinned reader held in a private field is
+     * outside the memory ceiling the library advertises — measured at 10.34 MiB per live pin, 32 pins holding
+     * 111.7 MiB against an 8 MiB configured bound. A pinned *generation number* has no such problem: an object is
+     * immutable, so eviction is harmless, and re-opening at the same number reopens the same bytes unless the name
+     * was purged and loaded again, which the pin's fingerprint then says. It also removes a memoized-rejection bug by
+     * construction — `install` forgets a promise that rejects,
+     * where a hand-rolled `this.reader ??= open()` cached the rejection for the life of the handle and made a pin
+     * the one read path with no resilience.
+     *
+     * **`status` is re-checked here**, on every open rather than once at pin time. A pin taken before a
+     * crypto-shred must not keep unwrapping a DEK the shred destroyed; a destroyed segment resolves no generation
+     * for anyone, and a pin's read of one fails. That holds from the pin's next open: a reader already open keeps
+     * the key it unwrapped until it is evicted or invalidated.
+     *
+     * **And the object is checked**, when the caller says which one it pinned. A generation number is not an
+     * identity (invariant 1): once a name is purged and loaded again, its new segment starts again at generation 0,
+     * so a pin of the old one would open the new one's object as its own, and read it beside the chunks it had
+     * already cached from the old one. The row's token cannot tell them apart, since it moves on every write, a
+     * publish included; the object's fingerprint can, and a pin whose object has been replaced fails, as one whose
+     * generation has been swept does. The memo is keyed so that two pins of one generation number in two
+     * incarnations never share a reader, with a registry or without one ({@link pinnedKey}).
+     */
+    private readerAt;
+    private openAt;
+    /**
+     * Read one chunk of a specific generation — the pinned read path. `held`, when given, is what a pin holds of the
+     * object, and the read fails if the generation is now another object.
+     */
+    getChunkAt(ref: ChunkRef, generation: number, held?: PinnedObject): Promise<Uint8Array | null>;
+    /**
+     * Throws the pin's NotFoundError if the object under `generation`'s key is no longer the one the pin holds. Its
+     * footer says, read with no key and no row ({@link CrbmReader.sameObject}), so an object written under a key this
+     * store lacks answers the same way, and so does one gone from its key. A replacement is kept
+     * ({@link replacedPins}), and a check under way is shared by the reads that ask at once; neither "gone" nor "the
+     * same" is kept, since either can change. A transient fault is rethrown, for the store's retries to see for what it
+     * is. A footer that cannot be read otherwise says nothing about which object is there, so this returns, and the
+     * caller's own error stands.
+     */
+    private throwIfReplaced;
+    /** Chunk keys of a specific generation — the pinned shape read. `held` as for {@link getChunkAt}. */
+    listChunkKeysAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<number[]>;
+    /**
+     * Per-chunk cardinalities of a specific generation — powers a pinned `count()` with no payload reads. `held` as
+     * for {@link getChunkAt}.
+     */
+    cardinalitiesAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<ReadonlyMap<number, number> | null>;
+    /** Grounded size of a specific generation. `held` as for {@link getChunkAt}. */
+    sizeOfAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<SegmentSize | null>;
+    /**
+     * Whether the segment is a registered name at all. With a registry that is exactly "a row exists" — including
+     * a row with no generation yet (minted by `setRetention` before the first load) and a `destroyed` tombstone,
+     * both of which are names somebody deliberately created. Without a registry there is no registration to
+     * consult, so the honest answer is whether the bucket holds any generation of it.
+     *
+     * Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
+     * tell apart.
+     */
+    exists(ref: SegmentRef): Promise<boolean>;
+    /**
+     * Evict a snapshot we just failed to read from — matched by its reader *promise*, so a concurrent call's
+     * fresher snapshot is never clobbered. The identity guard is the whole point: `install` replaces the entry
+     * wholesale, so comparing anything else would drop a good snapshot on the floor.
+     */
+    private dropStale;
+    private resolveLatest;
+    /** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
+    private openForTarget;
+    /**
+     * The current generation + its DEK wrappings: the registry's authoritative record (one strong read), or a
+     * `list` scan for the max generation when there's no registry (a registry-less source can only read cleartext
+     * — there's nowhere a wrapped DEK could live).
+     */
+    private resolveTarget;
+    /** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
+    private cryptoForRead;
+    getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
+    listChunkKeys(ref: SegmentRef): Promise<number[]>;
+    sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
+    cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
+    /**
+     * Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
+     * generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
+     * {@link NotFoundError}. Surfacing that as a query failure would turn a benign, recoverable race into a
+     * caller-visible error, so instead we drop the stale snapshot,
+     * re-resolve `currentGen`, and retry once — the read then serves the newer (committed, immutable) generation,
+     * a monotonic move forward — within one incarnation of the row. A name that was retired and re-created is a
+     * different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's token
+     * as its lineage rather than trusting the generation number (invariant 1).
+     *
+     * **Resolution and open are inside the retry, not before it.** Resolving `currentGen` and opening that
+     * generation's object are two backend round trips with a gap between them, so the object can be swept after
+     * the registry named it and before its tail is read — and that miss surfaces from awaiting the snapshot, not
+     * from `read`. Awaiting outside the retry left the *first* attempt unhealed on the one path GC actually
+     * races: `keep: 0`, which every id erasure passes, sweeps microseconds after the publish.
+     *
+     * A *second* NotFound is pathological (GC outrunning resolution) and propagates rather than fabricating an
+     * absent answer — never return a wrong result. `ifGone` is returned when the segment has no committed
+     * generation to serve: storage is empty, or the row was dropped or crypto-shredded, in which case reading empty
+     * is the documented outcome rather than a failure.
+     *
+     * **Bounded to exactly two resolve-and-open round trips.** That is a cost contract, not a detail: the retry
+     * now re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
+     * tight loop on a segment whose pointer names an object that is permanently absent — and an N-way
+     * `intersect` would do it N times. Gated in `tests/core/storage-source-heal-open.test.ts` by counting calls.
+     */
+    private withFreshSnapshot;
+}
+/**
+ * Write one immutable generation from in-memory bitmaps. Chunks are
+ * sorted ascending and empty bitmaps skipped (empty chunks are never stored). Returns the driver's
+ * `{ size, sha256 }` for the written object. Pass `options.crypto` to AES-256-GCM-encrypt the generation
+ * (built from the segment's DEK, with associated data bound to `(segment, generation)`).
+ */
+export declare function writeCrbmGeneration(driver: IStorageDriver, key: GenKey, chunks: Iterable<{
+    chunkKey: number;
+    bitmap: CodecBitmap;
+}>, options?: {
+    crypto?: CrbmCrypto;
+    clock?: Yielder;
+}): Promise<{
+    size: number;
+    sha256: string;
+}>;
+/** What {@link writeCrbmGenerationStream} wrote: the driver's `{ size, sha256 }` + a tally of the generation. */
+export interface StreamWriteResult {
+    readonly size: number;
+    readonly sha256: string;
+    /** Non-empty chunk keys written, in ascending order (for the post-write verify, since the stream is consumed). */
+    readonly chunkKeys: number[];
+    /** Total ids written. */
+    readonly cardinality: number;
+}
+/**
+ * Streaming variant of {@link writeCrbmGeneration} for a **constant-memory** rewrite (the erasure rewrite, a
+ * sorted load): consumes an **already-ascending** async stream of `{ chunkKey, bitmap }`, feeding each to the
+ * codec and freeing it, instead of materializing the whole generation. Paired with a streaming storage sink (S3
+ * multipart / LocalFs temp file), peak memory is ~one chunk + one part. Returns a tally so the caller can verify
+ * the re-opened object without re-iterating the (now-consumed) stream. Input **must** be ascending by `chunkKey`
+ * — the codec rejects an out-of-order chunk; empty bitmaps are skipped.
+ */
+export declare function writeCrbmGenerationStream(driver: IStorageDriver, key: GenKey, chunks: AsyncIterable<{
+    chunkKey: number;
+    bitmap: CodecBitmap;
+}>, options?: {
+    crypto?: CrbmCrypto;
+    clock?: Yielder;
+}): Promise<StreamWriteResult>;
+/**
+ * Point a segment's registry `currentGen` at `key.generation` — the publish step that makes a freshly-written
+ * generation the authoritative latest (so registry-aware readers see it). **Forward-only and idempotent:**
+ * if the registry has no row it creates one; if it's already at/ahead of `key.generation` it's a no-op (an
+ * out-of-order/duplicate publish never regresses the pointer); otherwise it advances via compare-and-swap,
+ * retrying a few times under contention. Separated from the Storage write so callers can publish atomically
+ * after the immutable object is durable (write-then-publish).
+ *
+ * **Throws** rather than returning, in two cases, because each would otherwise create a state no reader can
+ * recover from: a `destroyed` (crypto-shredded) row, where advancing the pointer would name an object encrypted
+ * under a shredded key; and an advance carrying **new** `wrappedDeks`, which would either strand that key or make
+ * the row advertise encryption over cleartext generations. Contention that never clears throws
+ * {@link WriteConflictError} after the retries.
+ *
+ * **`expectFrom` turns forward-only into read-modify-write.** Forward-only is the right rule for a writer whose
+ * content does not depend on what was current — a load computes its ids upstream, so publishing over a newer
+ * generation loses nothing the loader knew about. It is the WRONG rule for a writer that *derived* its content
+ * from a particular generation: the erasure rewrite streams generation `from` and clears one bit, so publishing
+ * it over a newer generation silently discards whatever that generation added. Passing `expectFrom` makes the
+ * publish land only while the pointer is still exactly there, and return `false` otherwise, so the caller can
+ * report `superseded` and re-derive. The compare-and-swap itself carries the token read in the same iteration,
+ * so the check and the write see the same row.
+ *
+ * **`expectToken` fences the LINEAGE, and a derived writer needs both.** A generation number identifies a
+ * generation only within one incarnation of a name: `nextGeneration` restarts at `0` once the row is purged and
+ * the bucket empty, so a name that is retired and re-created has a *different* segment wearing the *same*
+ * `currentGen`. `expectFrom` alone matched it — and an erasure rewrite then published one incarnation's content
+ * over another's, deleted the live objects with its `keep: 0` collection, and returned `erased: true`. The row's
+ * OCC token is the identity that survives this: {@link IRegistryDriver.delete} tombstones rather than unlinks,
+ * so "a later `create` still gets a fresh, greater token" and no token is ever reused across incarnations.
+ *
+ * Pass the token read alongside `expectFrom` and the publish lands only on the same row it was derived from.
+ * The check is deliberately **conservative**: the token also changes on writes that are not supersessions at
+ * all (a `setRetention`, a due-index reindex), so such a write makes a derived publish report `superseded` and
+ * the caller re-derive. That costs a re-run on a rare, unrelated write; the alternative costs a segment.
+ */
+export declare function publishGeneration(registry: IRegistryDriver, key: GenKey, options?: {
+    wrappedDeks?: readonly WrappedDek[];
+    expectFrom?: number;
+    expectToken?: Token;
+    /**
+     * Publish only while the segment still has **no registry row**.
+     *
+     * The fence for a caller whose decision rests on the row being ABSENT. `expectFrom` and `expectToken`
+     * cannot express it: both compare against a value read from a row, so when there was no row there is
+     * nothing to compare and both are simply omitted — leaving the publish a bare forward-only advance that
+     * happily lands over whatever appeared in the meantime.
+     *
+     * That gap was a silent wipe, not a theoretical one. A guarded load into a segment that did not exist yet
+     * read "no row", so its empty/`minRetained` bounds had nothing to judge and passed vacuously; a
+     * concurrent writer then created the row and published a thousand ids; and the guarded load published an
+     * EMPTY generation over them, reporting `published: true` with no reason. Reproduced through both
+     * `loadSegment` and the `*Into` verbs, which hit it far more often because materialising into a
+     * destination that does not exist yet is the ordinary first run of a pipeline.
+     */
+    expectAbsent?: boolean;
+}): Promise<boolean>;
+/** What a bulk-load wrote — the driver's `{ size, sha256 }` plus a summary of the built generation. */
+export interface BulkLoadResult {
+    /** Bytes written to the Storage object. */
+    readonly size: number;
+    /** The driver's content hash of the object. */
+    readonly sha256: string;
+    /** Distinct non-empty chunks written (≤ 65536). */
+    readonly chunkCount: number;
+    /**
+     * The DEK freshly minted for this generation, wrapped — present only when the segment is encrypted AND this
+     * call minted one (an existing segment reuses its key, which is already on the row).
+     *
+     * Returned so a caller that deliberately does **not** pass `registry` here — writing now and publishing later,
+     * which is how a load applies a guard while the old generation is still authoritative — can carry the key onto
+     * its own `publishGeneration`. Without it that publish stores no DEK and the generation it makes current is
+     * unreadable: the bytes are encrypted with a key nothing recorded.
+     */
+    readonly wrappedDeks?: readonly WrappedDek[];
+    /** Total distinct ids in the generation (post-dedup). */
+    readonly cardinality: number;
+    /**
+     * Whether this generation is now the segment's **current** one — `undefined` when no `registry` was wired, so
+     * there is no pointer and no publish step to report on.
+     *
+     * `false` means the object is durable but a **concurrent writer published a higher generation first**, so this
+     * one is an orphan that no reader will ever resolve. The write succeeded and the *load* did not: whatever this
+     * call was asked to make the segment contain, the segment does not contain. That is not a detail a caller can
+     * be left to infer — a `*Into` materialisation that reported the generation it wrote as "the destination's new
+     * current generation" on this path would state a plain untruth — so the flag is on the result and the verbs that
+     * promise a published generation check it.
+     */
+    readonly becameCurrent?: boolean;
+}
+/**
+ * **Bulk-load** a whole immutable generation from a flat id source — the batch "seed/rebuild" entry point.
+ * Unlike {@link writeCrbmGeneration} (which takes pre-grouped bitmaps), this
+ * consumes an arbitrary, **unsorted** stream of u32 ids (sync or async) and routes each into its chunk's
+ * Roaring bitmap as it goes. The input is consumed lazily and ids dedup on insert, so memory is bounded by
+ * the **built generation** — the in-memory Roaring representation of the *distinct* set (≤ one bitmap per
+ * non-empty chunk) — not by the input length: you can stream a billion duplicate-heavy ids and hold only the
+ * distinct result. Note this is `O(distinct set)`, **not** window-bounded like {@link SegmentEngine.intersect}:
+ * bulk-load holds the whole generation in RAM, which suits batch seed/rebuild jobs but not an
+ * unbounded-cardinality stream (a segment larger than RAM needs external-merge / pre-sorted input — later phase).
+ *
+ * Each id must be an integer in `[0, 2³²)` ({@link splitId} throws {@link ValidationError} otherwise) — the
+ * source is consumed lazily, so a bad id aborts mid-stream without writing a partial object (the driver
+ * commits only after the callback resolves). An empty source writes a valid empty generation.
+ *
+ * Writes a fresh full snapshot of a segment. The caller picks the generation number in `key` — `nextGeneration`
+ * computes the right one from the registry and the bucket — and re-using an existing generation throws
+ * {@link WriteConflictError} (write-once). Without a registry a `StorageChunkSource` serves the **highest**
+ * generation present, so a too-high number silently shadows real data; with one, `publishGeneration` decides.
+ */
+export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenKey, ids: Iterable<number> | AsyncIterable<number>, options?: {
+    registry?: IRegistryDriver;
+    keystore?: IKeystore;
+    requireEncryption?: boolean;
+    audit?: IAuditSink;
+    /** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
+    codec?: CodecInterface;
+    /**
+     * Injected clock. Supplying one makes bulk-load **cooperative**: it yields the event loop periodically so a
+     * long load does not stall everything else on the process. Without it the load still completes, just
+     * without yielding — which is the pre-existing behaviour, kept so this is purely additive.
+     *
+     * `@cloudbitmaps/roaring` supplies a real clock by default, so flavor users get cooperative behaviour with
+     * no wiring. `core/` cannot default it: it is timer-free by lint, which is exactly why waiting goes through
+     * this seam rather than `setTimeout`.
+     */
+    clock?: Clock;
+    /**
+     * Write the object but do **not** advance the pointer (default: publish when a `registry` is wired).
+     *
+     * For a caller that has to inspect what it wrote before deciding whether it should become current — a
+     * guarded {@link loadSegment} is the one in-tree case — because the check is only meaningful while the old
+     * generation is still authoritative. The `registry` is still required for an encrypted segment and still
+     * consulted: it is where an existing segment's DEK lives, and reusing that key is not optional. The wrapped
+     * DEK comes back on the result so the deferred publish can store it.
+     *
+     * A caller that defers the publish owns what it wrote: an unpublished object sits ABOVE `currentGen`, where
+     * generation collection deliberately never looks, so nothing reclaims it until a later generation above it is
+     * current.
+     */
+    publish?: boolean;
+}): Promise<BulkLoadResult>;
+/**
+ * Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
+ * The reader the write paths use to re-read what they wrote, and the erasure rewrite uses to stream the old
+ * generation; the engine's read path goes through {@link CrbmStorageChunkSource} instead, which caches these.
+ */
+export declare function openGenerationReader(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, options?: Omit<CrbmReaderOptions, 'crypto'>): Promise<CrbmReader>;
+/**
+ * Re-open a freshly written generation and assert it round-trips exactly what was streamed into it: the same
+ * per-chunk key set *and* the same total cardinality (on top of the codec's own per-chunk CRC + footer checks).
+ * `expected` is the streaming writer's tally (the stream is consumed, so it can't be re-iterated) — the key-set
+ * comparison catches a dropped/extra chunk that a cardinality-only check could miss when two errors cancel out.
+ * Throws {@link IntegrityError}: the object is on disk but must not be published.
+ */
+export declare function verifyGeneration(storage: IStorageDriver, key: GenKey, expected: {
+    readonly chunkKeys: readonly number[];
+    readonly cardinality: number;
+}, crypto: CrbmCrypto | undefined): Promise<void>;

package/dist/core/crypto.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Encryption seams — the **pure** interfaces `core/` depends on so the codec can encrypt/decrypt without
+ * importing `node:crypto` (lint-banned here, like `Clock`/`Rng`/drivers). Concrete AES-256-GCM lives outside
+ * `core/` (`src/drivers/crypto.ts`) and is injected. The envelope scheme: a per-segment DEK wrapped under an
+ * operator-held KEK, so discarding the DEK crypto-shreds the segment. Per-chunk AEAD over the `.crbm` payloads + index; a per-segment **DEK** wrapped (envelope
+ * encryption) under one or more **KEK**s by an {@link IKeystore}; the wrapped DEK(s) live in the registry, so
+ * crypto-shred = delete them.
+ */
+/** One AEAD result: a fresh per-message nonce, the ciphertext, and the authentication tag. */
+export interface AeadSealed {
+    readonly nonce: Uint8Array;
+    readonly ciphertext: Uint8Array;
+    readonly tag: Uint8Array;
+}
+/**
+ * Authenticated encryption bound to a single key (a segment's DEK). `aad` (associated data) is authenticated
+ * but not encrypted — the codec passes each chunk's `(namespace, segment, generation, chunkKey)` identity
+ * (the library derives it; an `Aead` implementation receives it), so a chunk's ciphertext can't be silently
+ * relocated to another segment/generation/chunk or
+ * the index, even by someone who can rewrite Storage objects. The seam is symmetric and self-framing-agnostic:
+ * the caller decides where the nonce/tag go (inline with a chunk, or in the footer for the index).
+ */
+export interface Aead {
+    /** Encrypt `plaintext`, authenticating `aad`. Returns a fresh nonce + ciphertext + tag. */
+    seal(plaintext: Uint8Array, aad: Uint8Array): AeadSealed;
+    /**
+     * Decrypt + verify. Throws {@link IntegrityError} on **any** mismatch (wrong key, tampered ciphertext/tag,
+     * or `aad` that doesn't match what was sealed) — never returns wrong-but-plausible plaintext.
+     */
+    open(sealed: AeadSealed, aad: Uint8Array): Uint8Array;
+}
+/**
+ * A DEK wrapped under one KEK. A segment stores a **list** of these (one per KEK it was wrapped under — e.g.
+ * an active KEK plus an offline recovery KEK), so losing one KEK isn't fatal and KEK rotation needs no
+ * data re-encryption. `keyId` names which KEK; `wrapped` is the opaque wrapped-key blob (base64). Carries no
+ * plaintext key material — safe to store in the registry next to the (encrypted) data.
+ */
+export interface WrappedDek {
+    readonly keyId: string;
+    readonly wrapped: string;
+}
+/**
+ * Envelope key management — the injected seam between the codec and however the operator holds key material
+ * (in-process BYOK by default; KMS/Vault as optional adapters later). Async so a remote keystore (KMS) fits
+ * the same interface. Never exposes raw key bytes to `core/`: it returns an {@link Aead} bound to the DEK.
+ */
+export interface IKeystore {
+    /**
+     * Mint a fresh per-segment DEK, wrapped under the active KEK (and any recovery KEK). Returns the wrapping(s)
+     * to persist in the registry plus an {@link Aead} to encrypt this segment's chunks with.
+     */
+    createDek(): Promise<{
+        wrapped: WrappedDek[];
+        aead: Aead;
+    }>;
+    /**
+     * Reconstruct the {@link Aead} from a segment's stored wrappings by unwrapping with any KEK currently held.
+     * Throws {@link KeyUnavailableError} if it holds none of the referenced KEKs (lost key, or a shredded
+     * segment whose wrappings were deleted).
+     */
+    openDek(wrapped: readonly WrappedDek[]): Promise<Aead>;
+}
+/**
+ * What the encrypting codec needs: the {@link Aead} for this segment's DEK plus a way to build the AAD for a
+ * given scope (`'index'` or a chunkKey). Built per (segment, generation) by the storage-source bridge so the
+ * codec itself stays segment-agnostic.
+ */
+export interface CrbmCrypto {
+    readonly aead: Aead;
+    aadFor(scope: number | 'index'): Uint8Array;
+}
+/**
+ * Build the AEAD associated-data binding a chunk (or the index) to its exact location:
+ * `v1 ‖ len(namespace) ‖ namespace ‖ len(segment) ‖ segment ‖ generation ‖ scope ‖ chunkKey`. Length-prefixed
+ * so distinct `(namespace, segment)` pairs can never collide. Pure (no crypto) — just the authenticated label.
+ */
+export declare function aadFor(ref: {
+    readonly namespace?: string;
+    readonly segment: string;
+}, generation: number, scope: number | 'index'): Uint8Array;