@cloudbitmaps/roaring 0.19.1 → 0.21.0

package/dist/index.d.ts

@@ -23,7 +23,7 @@
  * `checkConsistency`, `exportSegments` — reuse the store's own drivers, so you never re-pass them (they need the
  * store built with a backend). See the README and the getting-started guide.
  */
-import type { BudgetOption, GenerationEntry, GenerationMetadata, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, RollbackResult, Clock, StorageChunkSource, ConsistencyReport, CostReport, DropResult, EstimateInput, ExportManifest, ExportOptions, ExportSink, IAuditSink, IStorageDriver, IKeystore, IMetricsSink, StorageBackend, PricingProfile, RetentionPolicy, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, SetRetentionResult, PinnedAt, IdRange, SegmentRef, Workload } from '@cloudbitmaps/core';
+import type { BudgetOption, GenerationEntry, GenerationMetadata, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, RollbackResult, Clock, StorageChunkSource, ConsistencyReport, DropResult, ExportManifest, ExportOptions, ExportSink, IAuditSink, IStorageDriver, IKeystore, IMetricsSink, StorageBackend, RetentionPolicy, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, SetRetentionResult, PinnedAt, IdRange, SegmentRef } from '@cloudbitmaps/core';
 import type { SegmentInfo } from '@cloudbitmaps/core';
 import type { CombineExpr, CombineManyFeedRecord, CombineManyOperandStats, CombineManyOutputStats, CombineManyStats } from '@cloudbitmaps/core';
 import { deserializePortable } from './roaring-codec.js';
@@ -78,7 +78,7 @@ export interface CloudRoaringOptions {
      * lands and then loses its response would, replayed blindly, report its own write as a conflict. A write is
      * retried only where that is safe: a load's registry write that gets no answer is settled by reading the row,
      * and a throttled write-once object is sent again by the S3 and GCS drivers. Pass a partial policy to tune it — anything you leave out keeps its
-     * default — or `false` to turn the read retry off (e.g. if your injected client already retries). A GCS download is retried by
+     * default, except an omitted `maxDelayMs`, which is `baseDelayMs` when that is larger than 2000 ms — or `false` to turn the read retry off (e.g. if your injected client already retries). A GCS download is retried by
      * the GCS driver whatever this says.
      * Deterministic errors (`ValidationError`/`IntegrityError`/`WriteConflictError`/…) are never retried by this
      * layer.
@@ -117,13 +117,15 @@ export interface CacheOptions {
      * How long (ms) the store trusts a segment's resolved `currentGen` before re-resolving it on the next read
      * (default 2000) — the bound on read staleness after a load publishes a new generation. Applies when
      * `storage` is a **backend**, whose registry supplies the cheap `currentGen` read the refresh needs. Lazy — no
-     * timer; ≤ one registry read per segment per window, opening a new reader only when the generation actually
-     * advanced.
+     * timer; ≤ one registry read per segment per window, opening a new reader only when the row's pointer was
+     * renewed (a publish, a rollback, an erasure's rewrite, a status or key change); a lease or retention write does not.
      *
      * **While the registry cannot be read, the bound stretches.** A refresh that fails with a
-     * {@link TransientError} keeps serving the generation the reader holds, and the key it unwrapped, and is tried
-     * again 500 ms later (or after this TTL, if shorter), so the store converges within one retry of the registry
-     * answering. A refresh that fails with anything else, an access denial or a row that will not parse, is not
+     * {@link TransientError} keeps serving the generation of a segment still in the reader cache, and the key its reader
+     * unwrapped (a read that needs the key, where none was unwrapped before it, unwraps it from that segment's row as
+     * resolved before the outage), and is tried again 500 ms later (or after this TTL, if shorter), so the store converges
+     * within one retry of the registry answering. A segment the reader cache has let go is not served past its resolution's TTL: once that
+     * lapses, a read of it fails with the fault, as a cold read does. A refresh that fails with anything else, an access denial or a row that will not parse, is not
      * ridden out: the read that meets it throws that error, and the reader is dropped.
      *
      * `0` turns this timed refresh off, and so does wiring a bare `IStorageDriver`, which has no registry. That is
@@ -138,7 +140,13 @@ export interface CacheOptions {
     /**
      * Ceiling on how many segments' `.crbm` readers (each holding a parsed index) the store keeps open at once
      * (default 1024) — the steady-state memory bound for a long-running server that reads across many segments.
-     * Past it the least-recently-used segment's reader is evicted; re-opening it later is one cheap tail GET. A reader of a small generation, one whose whole object came with its tail read,
+     * Past it the least-recently-used segment's reader is evicted; re-opening it later is one cheap tail GET. With a
+     * timed refresh ({@link CacheOptions.genTtlMs} above 0, and a backend) the store also keeps up to 8 times this many
+     * segments' resolutions (what it read of each segment's row: never an unwrapped key, though the wrapped keys are in
+     * it) for `genTtlMs`, apart from their readers. An evicted segment is then reopened with no registry read, on the
+     * generation it had resolved, when a read needs its index or a chunk the chunk cache does not hold: `iterate` and the
+     * combines always do, and a `has()` of a cached chunk, a `count()` and a `stat()` do not, where the row has a summary
+     * the store can use. A reader of a small generation, one whose whole object came with its tail read,
      * also holds its chunk bytes, so a read of it makes no chunk request until the reader is evicted or the pointer refresh
      * moves it on; a store with no timed refresh (`genTtlMs: 0`, or no registry) keeps none.
      * Applies whenever the store builds its own read path — a backend or a bare `IStorageDriver`. A pre-built
@@ -146,14 +154,15 @@ export interface CacheOptions {
      */
     readonly readerMax?: number;
     /**
-     * Aggregate byte ceiling on what the open readers hold, their parsed `.crbm` indices, the metadata a
-     * generation carries, and the chunk bytes a reader keeps when its tail read returned a whole object whose chunks
+     * Aggregate byte ceiling on what the open readers hold: their parsed `.crbm` indices, the metadata a
+     * generation carries, a fixed 3.75 KiB for the objects each open reader holds around its index, and the chunk bytes a reader keeps when its tail read returned a whole object whose chunks
      * total at most this ceiling divided by {@link CacheOptions.readerMax} (default 64 MiB) — the byte half
      * of the memory bound, complementing the {@link CacheOptions.readerMax} *count* bound. A wide/dense segment's
      * parsed index can reach about 1.3 MB, so a count-only bound could let the open readers pin over a GB and blow a small
-     * heap (e.g. a 128 MB Lambda); this evicts the least-recently-used reader once the summed index footprint
-     * would exceed the ceiling — whichever of the count/byte bounds binds first. Lower it for memory-tight
-     * deployments that read across wide segments. Applies whenever the store builds its own read path.
+     * heap (e.g. a 128 MB Lambda); this evicts the least-recently-used reader once the summed weight
+     * would exceed the ceiling — whichever of the count/byte bounds binds first. A sixteenth of it (4 MiB by default)
+     * bounds the segments' resolutions a store with a timed refresh keeps besides. Lower it for memory-tight
+     * deployments that read across wide segments. A ceiling below two readers' weight (about 7.7 KB for tiny ones) keeps one reader open at a time. Applies whenever the store builds its own read path.
      */
     readonly readerMaxBytes?: number;
 }
@@ -181,7 +190,8 @@ export interface EncryptionOptions {
  * {@link CloudRoaringOptions.retry} — a partial {@link RetryPolicy} plus the retry callback.
  *
  * Partial on purpose, so tuning one field does not mean restating all five. Anything omitted here keeps its
- * {@link DEFAULT_RETRY_POLICY} value.
+ * {@link DEFAULT_RETRY_POLICY} value, except that an omitted `maxDelayMs` is the `baseDelayMs` when that is larger
+ * than the default cap.
  */
 export interface RetryOptions extends Partial<RetryPolicy> {
     /**
@@ -242,8 +252,9 @@ export interface SubjectErasureEntry {
     /** The generation written without the id (present whenever one was written). */
     readonly generation?: number;
     /**
-     * Why the id was NOT erased from this segment, when `erased` is false. `'superseded'` — the pointer moved while
-     * the call was in flight, by a load, another erasure or a rollback, so **this call** did not erase the id;
+     * Why the id was NOT erased from this segment, when `erased` is false. `'superseded'` — another writer (a load,
+     * another erasure or a rollback) moved the pointer, or replaced an object the call meant to delete, while the call
+     * was in flight, so **this call** did not erase the id;
      * re-run against the new generation, which erases it if it is still there and reports nothing for the segment
      * if the racing writer already removed it. `` `error: <message>` `` — an isolated per-segment fault
      * (per-segment faults are recorded so one segment can't discard the whole ledger); re-run after fixing the
@@ -455,7 +466,8 @@ export interface MaterializeManyDryRun {
 export interface MaterializeResult {
     /**
      * The generation written. Present even when refused: the refusal deleted that object, unless the destination's
-     * row changed while it was writing, which leaves it in the bucket.
+     * row changed while it was writing, which leaves it in the bucket, an orphan a later load's collection takes; until
+     * then a rollback onto it needs `allowUnknown`.
      */
     readonly generation: number;
     /**
@@ -474,8 +486,9 @@ export interface MaterializeResult {
      * What the destination held when the guard judged it — `null` when it had no current generation, **or when
      * no bound needed it**. It is taken only when a bound will use it: `allowEmpty: true` with neither
      * `guard.minRetained` nor `guard.maxGrowth` skips it, and this is `null` even though `dest` was non-empty. When the destination's row
-     * carries a summary of its current generation the count comes from the row and the object is not read; a row
-     * written before rows carried a summary costs one object-header fetch.
+     * carries a summary of its current generation the count comes from the row and the object is not read; a row with
+     * none (one a rollback left when it could not open the key, one a registry of your own stores without one, or one this store cannot use) costs
+     * one object-header fetch.
      */
     readonly cardinalityBefore: number | null;
     /** Non-empty chunks in the generation. */
@@ -533,10 +546,11 @@ export declare class CloudRoaring {
     private lifecycleDeps;
     /**
      * Get a handle to a segment. Validates the name (non-empty, well-formed, within the encoded-length cap). The options
-     * are `{ namespace }` and nothing else: anything else is refused with {@link ValidationError}, by name, since a
-     * misspelt `namespace` would otherwise address the default namespace. An `expiresAt` among them is refused with its own
-     * message: a handle carries no deadline, and one left unread would serve the data past it. A key whose value is
-     * `undefined` is read as absent, so a spread of options keeps working.
+     * are a plain object holding `{ namespace }` and nothing else: any other own key, enumerable or not, is refused with
+     * {@link ValidationError}, by name, since a misspelt `namespace` would otherwise address the default namespace, and a
+     * deadline such as `expiresAt` would be left unread. A class instance, or an object built on another, is refused too,
+     * since it can carry an option no scan of its own keys sees. A key whose value is `undefined` is read as absent, so a
+     * spread of options keeps working.
      */
     segment(name: string, options?: SegmentOptions): Segment;
     /**
@@ -551,13 +565,20 @@ export declare class CloudRoaring {
      * of it. A materialisation is a load whose ids happen to come from a combine instead of from upstream, so
      * everything `load()` learned the hard way applies unchanged: the generation is written UNPUBLISHED, the
      * guard runs while the old generation is still authoritative, the publish is fenced as above, and a refused
-     * object is reclaimed only when a re-read finds the row unchanged (the same token) or gone (hard invariant 1:
-     * deleting it after a purge-and-recreate would put a live row over a missing generation).
-     *
-     * That last check narrows the window rather than closing it: the row read and the delete are two round
-     * trips, and `IStorageDriver` has no conditional delete to make them one. the collection a load runs carries
-     * the same residual. The failure it leaves is an orphan object, which costs storage until
-     * something collects it — deliberately the cheaper side of the trade.
+     * object is reclaimed only when a re-read finds the row unchanged (the same token, or one that differs only in
+     * its leases), `destroyed`, or gone, or finds key material on it when the load wrote cleartext; under any other
+     * row it is kept (hard invariant 1: deleting it after a purge-and-recreate would put a live row over a missing
+     * generation).
+     *
+     * Where the row is gone, or keyed over a cleartext write, the object's footer must match what the load wrote, and
+     * the delete passes the version that read reported, so on a storage driver that reports `conditionalDelete` an
+     * object put under the number since is kept. Where the row is unchanged or `destroyed`, the check narrows the
+     * window rather than closing it: the row read and the delete are two round trips, and the delete is by number,
+     * since the write reports no version of the object it made. The other deletes a load leads to are by number too: its
+     * collection deletes what a listing, or the row's list of kept generations, names below the pointer, and a drop's
+     * sweep and a tombstoned segment's collection delete what a listing names under a `destroyed` row. The failure the
+     * checks leave when they refuse is an orphan object, which costs storage until something collects it —
+     * deliberately the cheaper side of the trade.
      *
      * Written and published in one step, with no guard, an empty combine — a typo'd operand, an `exclude` that
      * swallowed everything, an operand that had not loaded yet — would silently replace `dest` with an empty
@@ -746,10 +767,14 @@ export declare class CloudRoaring {
      * can still hold it, and so can one above the pointer after a {@link CloudRoaring.rollback}, which the rollback
      * could make current again. Each holder is deleted — below the pointer by a `keep: 0` collection, above it one
      * by one, re-proved against the row first — and the generations above the pointer that never held the id stay
-     * as rollback targets. An entry says `erased: true` only once the call has listed the bucket and read what is
-     * left: **no generation of the segment holds the id**. The returned per-segment record is your **erasure ledger** —
-     * persist it / route it to your audit sink as the proof of deletion (a `segment.rewrite` audit event is also
-     * emitted per rewrite when you pass `audit`).
+     * as rollback targets, but for any object no read of the segment can open, which goes whatever the id. An entry
+     * says `erased: true` only once the call has listed the bucket and read what is left: **no generation of the
+     * segment holds the id**. The returned per-segment record is your **erasure ledger** —
+     * persist it / route it to your audit sink as the proof of deletion (when you pass `audit`, an audit event is also
+     * emitted per segment erased: `segment.rewrite` for a rewrite, `segment.collect` where only other generations held
+     * the id and nothing was rewritten). A segment where the erasure deleted only objects no read of it can open (a first
+     * load's sealed under a key no row holds) found the id in none, so the ledger does not list it; the audit sink gets
+     * `segment.collect` for it with no `fromGeneration`, which is the only record of that deletion.
      *
      * Uses the backend's **own** two halves, so the membership check and the rewrite provably run
      * over the same generation. Requires the store built with a **backend** (throws
@@ -771,13 +796,21 @@ export declare class CloudRoaring {
      * an error, read off the row: a moved pointer is a `'superseded'` entry, which a re-run settles, and a segment
      * that a concurrent `dropSegment` tombstoned or a retention sweep purged is left out of the ledger. A tombstoned
      * segment is still searched: a cleartext object left under it that holds the id is deleted, with everything under
-     * the tombstone, and the entry reads `erased: true`.
+     * the tombstone, and the entry reads `erased: true`. So is a segment whose row names no generation yet (a retention
+     * policy set before its first load): a first load's object that holds the id is deleted, once the row's `pointerId`
+     * is renewed so that the load, if still running, is refused at its publish, and the entry reads `erased: true`. An
+     * encrypted first load's object there cannot be searched, so it is deleted whatever the id: on an encrypted store,
+     * an erasure of any id refuses an encrypted first load in flight that has written its object onto a segment made
+     * ahead of its data. Each such delete, and each of a holder above the pointer, names the object the erasure read:
+     * on a storage driver that reports `conditionalDelete`, another erasure of the id that deletes it first, followed by
+     * a load that takes the number freed and publishes, does not lose that load's generation to this erasure's delete,
+     * which is refused, and the entry says what the row says; on one that does not, it can.
      *
      * **Read `note` on any `erased: false` entry — the two reasons mean different things.** `'superseded'` means
-     * another writer (a load, another erasure, or a rollback) moved the pointer mid-call, so **this call** did not
-     * erase the id. Re-run: it erases the id if it is still there, and lists nothing for the segment if a racing
-     * erasure of the same id already removed it. Do not read `'superseded'` as "the id is still present" —
-     * read it as "not done by this call, and the re-run settles it".
+     * another writer (a load, another erasure, or a rollback) moved the pointer, or replaced an object the call meant to
+     * delete, mid-call, so **this call** did not erase the id. Re-run: it erases the id if it is still there, and lists
+     * nothing for the segment if a racing erasure of the same id already removed it. Do not read `'superseded'` as
+     * "the id is still present" — read it as "not done by this call, and the re-run settles it".
      * `` `error: …` `` is a per-segment fault (caught so one segment can't discard the whole ledger) and it can land
      * on either side of the publish: if the rewrite had not published, the id is still there and a re-run erases
      * it; if the publish succeeded and only the **collection** of the old generation failed, the id is already
@@ -812,6 +845,8 @@ export declare class CloudRoaring {
      * What the bucket holds, not what the segment has ever been — collection deletes superseded objects, so this is
      * the grace window plus whatever has not been collected yet. It is the set {@link CloudRoaring.rollback} can
      * choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
+     * Each entry says whether the row knows the generation (`known`): a known generation needs no `allowUnknown`; an unknown
+     * one does, except below a row's pointer when the row has no usable record or the target is older than a full one reaches.
      *
      * Needs a backend.
      */
@@ -884,35 +919,58 @@ export declare class CloudRoaring {
      * (`segment.rollback`), because every other pointer move can be reconstructed from "a load happened" and this
      * one cannot.
      *
-     * It refuses rather than guesses: a generation not in the bucket (collected, or never written) throws
-     * `NotFoundError` naming what *is* available, a crypto-shredded segment throws
-     * {@link ValidationError} because every generation of it is unreadable, and a target that is cleartext under an
-     * encrypted segment, or encrypted under a cleartext one, throws {@link IntegrityError} from one tail read of it,
-     * because every read would refuse it. Rolling to the generation already current is a no-op that reports itself.
+     * It refuses rather than guesses. The row records, in `knownGens`, the generations its loads and rollbacks made
+     * current, each with the footer checksum of its object, and a rollback moves only onto one of those: a generation the
+     * row does not know (a load that wrote its object and died before the publish, a refused load's object, an object
+     * sealed under a key no row holds, a load published after the point a registry was restored to) throws
+     * {@link ValidationError} before any request to storage, and a known one whose object carries another checksum than
+     * the recorded one throws {@link IntegrityError}, with or without a keystore, since the footer is in the clear. A
+     * generation not in the bucket (collected, or never written) throws `NotFoundError` naming what *is* available, a
+     * crypto-shredded segment throws {@link ValidationError} because every generation of it is unreadable, and a target
+     * that is cleartext under an encrypted segment, or encrypted under a cleartext one, throws {@link IntegrityError} from
+     * one tail read of it, because every read would refuse it. Rolling to the generation already current is a no-op that
+     * reports itself, once one footer read finds its object in the bucket (and the one the row recorded); when the object is
+     * gone it throws `NotFoundError` naming what is.
      *
      * The same tail read (and a range read when the target's index is longer than it) gives the target's id count and
      * metadata, which the rollback writes into the row with the pointer, so a reader that sees the target as current sees
      * what describes it. The store's keystore opens an encrypted target for this; without it, or when it cannot open the
-     * key for any reason, the segment still rolls back and the row carries no summary of the target.
+     * key (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out
+     * as `AbortSignal.timeout()` and the AWS SDK report a timeout), a rollback onto a known generation still happens,
+     * opening nothing, and the row carries no summary of the target. Any other failure of the keystore is
+     * thrown. A swap of the pointer that gets no answer is settled by reading the row: one that landed, the row being the
+     * one the swap leaves, is checked as any is, and otherwise its `TransientError` is thrown. After the swap the target's
+     * footer is read once more, and the pointer put back if the object is gone or is another.
      *
      * It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
-     * and are then *above* `currentGen`, where collection never looks. They remain until loads pass them (the first
-     * load whose number one of them holds numbers above them all, and collection then keeps the newest `keep` of what is
-     * below its pointer), {@link CloudRoaring.dropSegment}
-     * deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and only those holding
-     * the id when the current generation does not. An operator who has just undone a bad load should not have the
+     * and are then *above* `currentGen`, where collection never looks, and still known to the row, so rolling forward onto
+     * one needs no flag. They remain until loads pass them (the first load whose
+     * number one of them holds numbers above them all, and collection then keeps `keep` of what is below its pointer, the
+     * generation that load superseded first), {@link CloudRoaring.dropSegment}
+     * deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and, when the
+     * current generation does not hold the id, only those holding it and any no read of the segment can open, whatever
+     * the id. An operator who has just undone a bad load should not have the
      * evidence collected out from under them, while a rollback target that still holds erased data would make the
      * erasure undoable.
      *
-     * A target **above** the pointer needs `{ allowForward: true }`, and is refused with {@link ValidationError}
-     * without it: that is also where objects live that were never published, such as a load that wrote its object
-     * and died before the publish. Undoing an earlier rollback is what the opt-in is for.
+     * `{ allowUnknown: true }` moves onto a present object the row does not vouch for, or one whose checksum differs from
+     * the recorded one, as long as it is the segment's kind of object, opens, and names the generation, and records it as
+     * known (over a usable record; a row with none stays without one). On an encrypted segment, outside the two cases the pointer alone decides (below), it also needs the key: with no keystore, or one that holds none of the key's KEKs or
+     * one that does not unwrap it, it is refused with `KeyUnavailableError` before the target is read, since an object
+     * sealed under a key no row holds passes every check that needs no key, and a key service that did not answer in
+     * time is thrown as `TransientError`, to retry. Every move the row did not vouch for is audited with `unknown: true`.
+     * Two cases are decided by the pointer alone: a target below it needs no flag, no key and no checksum check, and the
+     * move is audited the same way. They are a row with no usable `knownGens` (a registry that drops the field, or a pointer
+     * moved by hand), and a target numbered below the bound of a full record (128 entries or more, leased ones included),
+     * which is its least recently current entry that no lease names (an `*Into` destination keeps every generation in the
+     * bucket and names only the newest 128). A generation a publish with a smaller `keep` no longer names but that stays in
+     * the bucket (a lease spared it, or its delete failed) needs `allowUnknown` until a later listing collects it.
      *
      * Needs a backend.
      */
     rollback(ref: SegmentRef, toGeneration: number, options?: {
         audit?: IAuditSink;
-        allowForward?: boolean;
+        allowUnknown?: boolean;
     }): Promise<RollbackResult>;
     /**
      * **Replace this segment's contents** with `input`, as one new immutable generation, and make it current.
@@ -962,10 +1020,16 @@ export declare class CloudRoaring {
      * published, a retention change, a rollback or an erasure wrote the row, or the row was deleted.
      *
      * A refused load deletes the object it wrote before returning — it sits above `currentGen`, where generation
-     * collection deliberately never looks — but only while the segment's registry row is unchanged or gone. Once
-     * another write has changed the row, the generation number it holds may name another incarnation's live object,
-     * so it leaves the orphan rather than risk deleting live data. No row's list names it, so it takes no place in a
-     * window, and a listing deletes it once a later generation is current above it.
+     * collection deliberately never looks — but only while the segment's registry row is unchanged (or differs only in
+     * its leases), `destroyed` or gone, or has key material when the load wrote cleartext, which has no place in an
+     * encrypted segment's bucket. Under a row that is gone or keyed, one read of the object's footer must match what the
+     * load wrote first. Once another write has otherwise changed the row, the generation number it holds may name another
+     * writer's object, and a footer names bytes, not the load that wrote them, so it leaves the orphan rather than risk
+     * deleting live data. A publish never adds the orphan to the row's list, and a load's listing deletes it once a
+     * generation above it is current, unless the row's list names it; where the row records none (after a rollback, or
+     * with a `keep` above 64), that listing's window can keep it and, at a `keep` up to 64, record it, and the publishes
+     * after carry it until the window pushes it out. Until it goes, a rollback onto it is refused unless `allowUnknown` is passed, since the list never names it (a list seeded
+     * over a row with none can); with the override it publishes what no load published.
      *
      * **Collection is by name at any `keep` up to 64.** The segment's row records the generations a load keeps, so a load
      * that found nothing above the pointer deletes the generations its publish pushed out of the window and lists
@@ -1139,8 +1203,8 @@ export declare class CloudRoaring {
      * Pass `purgeTombstones: false` to keep every tombstone, that row included. On a backend whose registry reports
      * `conditionalDelete` (AWS S3, Azure Blob, the local filesystem and memory, by default; GCS when you set `conditionalDelete: true`)
      * the purge removes the row from the bucket for good, by a delete the store applies only to the version it judged,
-     * so a full sweep reads what is live or inside its grace rather than every name a namespace ever held; elsewhere,
-     * and for a row a release before 0.12 wrote, it leaves a tombstone. Each retirement files a pointer in the due index
+     * so a full sweep reads what is live or inside its grace rather than every name a namespace ever held; elsewhere it
+     * leaves a tombstone. Each retirement files a pointer in the due index
      * under the day its tombstone's grace ends, so `scan: 'index'` purges as well as retires.
      *
      * Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
@@ -1173,7 +1237,10 @@ export declare class CloudRoaring {
      *   reader converging, you need to signal them — this is the call to make when your own fan-out delivers.
      *
      * It also forgets any replacement this store found of a pin's object, so once it is called after a restore puts that
-     * object back, the pin reads it again.
+     * object back, the pin reads it again. A read of the segment already running on this store, a pin's and a combine's
+     * with a pinned operand included, caches nothing more of what it began reading before the call, and a pinned one
+     * opens its object again before its next chunk, failing with `NotFoundError` if the object is gone or replaced, or
+     * its row is gone or destroyed.
      *
      * Synchronous, best-effort, and safe to call for a segment this store has never read.
      *
@@ -1189,7 +1256,8 @@ export declare class CloudRoaring {
      *
      * The pinned handle gets its own engine but **shares the store's chunk cache**, under keys of its own: the
      * pinned view reports the version captured at pin time, marked as a pin's, so its decoded chunks are never
-     * those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}).
+     * those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}). It
+     * shares the store engine's invalidations too, so the store's writes reach a pinned read already running.
      */
     private pinSegment;
     /** 16 hex digits from the store's `Rng`, which names the holder of one lease: unique, not secret. */
@@ -1198,7 +1266,8 @@ export declare class CloudRoaring {
      * An engine reading every segment in `pins` at its pinned generation and everything else live, with the store's
      * retries, so a transient fault on a pinned read is retried as it would be on a live one.
      *
-     * It **shares the store's chunk cache and its memory bound**, under keys of its own for each pinned segment.
+     * It **shares the store's chunk cache and its memory bound**, under keys of its own for each pinned segment, and the
+     * store engine's invalidations, so an invalidation of a pinned segment reaches a read of the pin already running.
      */
     private engineWithPins;
     /**
@@ -1232,10 +1301,17 @@ export declare class CloudRoaring {
      * A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
      * answers the number its row records, while a read of the object throws, so the count alone never shows it. With
      * `summaries: true` it also opens each segment's current object (one tail read each) and reports a segment whose
-     * row's summary says another id count or metadata than the object holds as `summary-mismatch`, which is what a
+     * row's summary names another object by its fingerprint, or says another id count or metadata than the object holds,
+     * as `summary-mismatch`, which is what a
      * restore of the registry from another point than the bucket leaves a count to answer. A sealed summary is held
      * against its object when this store has the key, and counted in `summariesUnchecked` when it does not.
      *
+     * With `generations: true` it also holds every generation a row's ledger of known generations names, the current one
+     * included, against its object: one footer read each (104 bytes, no key, not retried), reporting
+     * `known-generation-mismatch` with the `generation` where the object under that number is not the one the row
+     * recorded. A named generation that is not in the bucket is not reported. A row with a pointer whose ledger cannot
+     * be used is counted in `generationsUnchecked`, and none of its objects is read.
+     *
      * It holds the registry rows it enumerates resident, at most 250,000 of them, and throws
      * `BudgetExceededError` past that rather than report a partial scan as a whole one. This method takes no
      * ceiling of its own: narrow the scan with `namespace`, or call `runConsistencyCheck` over the backend's
@@ -1245,6 +1321,7 @@ export declare class CloudRoaring {
         namespace?: string;
         concurrency?: number;
         summaries?: boolean;
+        generations?: boolean;
     }): Promise<ConsistencyReport>;
     /**
      * **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
@@ -1259,13 +1336,6 @@ export declare class CloudRoaring {
      * CLI wraps this with a filesystem sink.
      */
     exportSegments(sink: ExportSink, options?: ExportOptions): Promise<ExportManifest>;
-    /**
-     * Planning cost estimate — pure, no instance/data needed: sizing, sales, what-if. For a real, grounded report
-     * from live segment sizes, use `store.segment(name).costReport()`. See {@link CostReport}.
-     * @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
-     * needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
-     */
-    static estimateCost(input: EstimateInput): CostReport;
 }
 /**
  * Options common to every chunk-aligned combine (`intersect` / `union` / `andNot`), and the `*Into` verbs.
@@ -1282,7 +1352,7 @@ export interface BaseCombineOptions extends IdRange {
      * is larger), about `concurrency × operands` MiB are held, whatever the segment's size. The stream opens 4 ranges wide
      * (or `concurrency`, if lower) and widens as ranges are taken, so a read that stops early has asked for little past
      * where it stopped, and a read that fits one range makes one request however many chunks it needs. A source that
-     * reads chunk by chunk (a custom one) is read as before: `concurrency` chunk keys at once, opening 8 wide. A running
+     * reads chunk by chunk (a custom one) is read `concurrency` chunk keys at once, opening 8 wide. A running
      * stream holds its operand's reader outside the reader cache's bounds until it ends or the segment moves on.
      */
     readonly concurrency?: number;
@@ -1368,18 +1438,32 @@ export interface MaterializeOptions extends CombineOptions {
      */
     readonly keep?: number;
 }
+/**
+ * An id stream as an iterator: the ids ascending, driven by `for await` or by hand with `next()`, `return()` and
+ * `throw()`, as a generator is, on a handle that holds a lease as on one that does not. It is its own iterator and
+ * single-use. {@link Segment.everyNth} returns this; the streams of {@link IdStream} add `batches()`.
+ */
+export interface IdIterator extends AsyncIterableIterator<number> {
+    /** Ends the read, as a generator's `return` does, and resolves done with `value`. */
+    return(value?: unknown): Promise<IteratorResult<number, unknown>>;
+    /** Throws `err` into the read, as a generator's `throw` does: it rejects with `err` unless the read handles it. */
+    throw(err: unknown): Promise<IteratorResult<number, unknown>>;
+}
 /**
  * What a streaming read returns: the ids, ascending, one at a time under `for await`, or one chunk at a time from
  * {@link IdStream.batches}.
  *
- * It is an `AsyncIterable<number>`, so every consumer of an id stream takes it as it always did. A live read is the
+ * It is an `AsyncIterable<number>`, so every consumer of an id stream takes it as an async iterable, and an
+ * {@link IdIterator}, so `next()`, `return()` and `throw()` are typed. A live read is the
  * engine's async generator itself, with `batches` attached: `for await` it, or drive it with `next()`, `return()` and
- * `throw()`, exactly as before, and it is single-use (a second `for await` over it yields nothing). `batches()` is a
+ * `throw()`, and it is single-use (a second `for await` over it yields nothing). A handle that holds a lease returns a
+ * stream of the same shape, which checks the lease as it reads. `batches()` is a
  * separate, new read: it starts when called, fetches its chunks afresh and charges the per-op budget again, whether or
  * not the per-id stream was read, and reading both is two reads (invariant 3 applies to each separately). Nothing is
- * fetched until a read is first pulled.
+ * fetched until a read is first pulled. A stream pulled with `next()` and dropped without `return()` is held by nothing
+ * in the store, so it is garbage once the caller lets it go.
  */
-export interface IdStream extends AsyncIterable<number> {
+export interface IdStream extends IdIterator {
     /**
      * The same ids as the per-id stream, in the same order, one `Uint32Array` per chunk: each array ascending, holding
      * every id of one chunk the read yields (a range trimmed at its edges exactly as the per-id stream trims it), and
@@ -1432,13 +1516,22 @@ export interface PinAt {
     /** The fingerprint of that generation's object, from the earlier pin's `pinnedAt.fingerprint`. */
     readonly fingerprint: string;
 }
-/** What {@link Segment.stat} answers: the generation a handle reads, its id count and its metadata. */
+/**
+ * What {@link Segment.stat} answers: the generation a handle reads, its id count, its metadata and the size of its
+ * object.
+ */
 export interface SegmentStat {
     /** The generation read, or `null` when the segment has none. */
     readonly generation: number | null;
     readonly cardinality: number;
     /** The metadata the generation was loaded with; absent when it has none. */
     readonly metadata?: GenerationMetadata;
+    /**
+     * The generation's object in storage, in bytes: what the segment stores. From the registry row's summary, which
+     * records the object's size with its checksum, or from the object's footer when the row has no summary to use.
+     * `null` when the segment has no generation, or when the store's source cannot report a size.
+     */
+    readonly sizeBytes: number | null;
 }
 /**
  * A handle bound to one segment — the read verbs, plus the three `*Into` verbs that write a **new generation**
@@ -1529,11 +1622,16 @@ export declare class Segment {
      * - `eraseSubject` invalidates a pin of each segment it scans that is not already destroyed.
      *
      * An invalidated pin opens its object again, and fails if that object is gone or replaced, or its row is gone or
-     * destroyed.
+     * destroyed. A read of the pin already running when the invalidation comes, a combine on this store with the pin as
+     * an operand included, looks the pin up again before it hands out its next chunk: it carries on with what it holds
+     * where the object is still there, and fails with `NotFoundError` where it is gone. It caches nothing more of what it
+     * began reading before the invalidation, so the pin's next read opens the object again too.
      * Anything else leaves the pin as it is: after a `destroySegment` beside this store, or an erasure, a drop or a
      * retirement through another store, in this process or another, the pin answers from what it holds until this
      * store's reader cache evicts the pin's reader and its chunk cache evicts the chunks the pin decoded, or
-     * {@link CloudRoaring.invalidate} is called on this store. Where the
+     * {@link CloudRoaring.invalidate} is called on this store. A combine on another store that takes the pin as an
+     * operand reads it through that store's reader and chunk cache, which only that store's own writes and its
+     * `invalidate` reach. Where the
      * object the pin reads has been deleted, by an erasure, a drop or a sweep, a chunk the pin has not cached fails at
      * once. No timed refresh bounds any of that, as none bounds anything else a pin holds.
      *
@@ -1586,6 +1684,8 @@ export declare class Segment {
      * the engine, so no operand, exclude or cached reader can answer past a lease.
      */
     private assertLeases;
+    /** Throw this handle's lease error, if it has one. The calls that return a value instead of throwing use {@link leaseError}. */
+    private assertLease;
     /**
      * Reopen a generation a pin named earlier, as a pinned handle like the one {@link Segment.pin} returns, held at
      * `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
@@ -1629,18 +1729,23 @@ export declare class Segment {
     /** `result` as is, or, when a handle of the call holds a lease, checking that lease each time it reads a chunk. */
     private guarded;
     /**
-     * Time an op with the injected clock and emit an `op` metric on completion (success or throw). Skipped
-     * entirely when no sink is wired, so the default path pays nothing.
+     * Time an op with the injected clock and emit an `op` metric on completion (success or throw). With no sink wired it
+     * reads no clock and emits nothing, but a call still passes through this async frame; `has()` and `count()` return the
+     * engine's promise directly instead.
      */
     private timed;
-    /** Membership: one chunk — the cache, else one ranged GET. Throws {@link ValidationError} on a bad id. */
+    /**
+     * Membership: one chunk. A cold resolution of the segment is one registry read and a warm one none; then a cached
+     * chunk answers with no request, and an uncached one is one range read, after a tail read when no reader of the
+     * generation is open, unless that tail carried the chunk. Throws {@link ValidationError} on a bad id.
+     */
     has(id: number): Promise<boolean>;
     /**
      * Cardinality of the generation this handle reads: **one registry read when cold, none when warm, and no read of the
      * object**, with **zero payload reads**. The registry row records the id count of the generation it names, written
      * by the write that made it current, and a count answers from that. A row with no summary it can use (one written
-     * before rows carried it, one that names another generation, a sealed one that does not open) sends the count to the
-     * `.crbm` index, summed with a tail read of the object. A segment whose pointer names a missing object (a torn
+     * without it, as a registry of your own may, one that names another generation, a sealed one that does not open)
+     * sends the count to the `.crbm` index, summed with a tail read of the object. A segment whose pointer names a missing object (a torn
      * restore) still counts the row's number, while a read of the object throws; `checkConsistency` is what finds it.
      *
      * What this trusts: the row's summary, or the index's sum, and no payload is decoded to confirm it. The summary is
@@ -1655,17 +1760,30 @@ export declare class Segment {
      */
     count(): Promise<number>;
     /**
-     * What the generation this handle reads is, from one resolution: its number, its id count and the metadata it
-     * was loaded with (absent when it has none). It is what answers {@link Segment.count}, so the three describe one
-     * generation and cannot straddle a publish. One registry read when cold, none while warm, and none on a pinned
-     * handle, which answers for the generation it pinned. A segment with no generation answers
-     * `{ generation: null, cardinality: 0 }`.
-     *
-     * Trust is as for `count()`: the registry row's word, not confirmed against the object until the object is
-     * opened, when a disagreement makes this process stop using that row's summary.
+     * What the generation this handle reads is, from one resolution: its number, its id count, the metadata it was
+     * loaded with (absent when it has none), and `sizeBytes`, its object's bytes in storage. All four come from the
+     * registry row when it has a summary this store can use, as `count()`'s do: the summary records the object's
+     * fingerprint, its size with its checksum. That is **one registry read when cold, none when warm, and no read of the
+     * object**. With no summary it can use, they come from the generation's object, opened once: its footer and index,
+     * with no payload read, which is a tail read of the object more when cold (and a range read for an index longer than
+     * the tail) and keeps the generation in the reader cache as a read does. On a pinned handle it answers for the
+     * generation it pinned, from the pinned object. A segment with no generation answers
+     * `{ generation: null, cardinality: 0, sizeBytes: null }`, and a store whose source cannot report a size answers
+     * `sizeBytes: null`. A pointer that names a missing object (a torn restore) still answers from the row's summary, as
+     * `count()` does, while a read of the object throws `NotFoundError`; with no summary to use, `stat()` throws it too.
+     * `checkConsistency` is what finds such a row.
+     *
+     * Trust is as for `count()`: what the row's summary says is believed until the object is next opened. Every open of
+     * the generation holds the object to the fingerprint the summary records, so another object under the number is
+     * refused when it is opened, and holds its count against the summary's; a disagreement in count makes this process
+     * stop using that summary, so `stat()` then answers from the object, whose index is checked for internal consistency
+     * and whose count is the index's sum. A reader already open is not checked again until the row changes, it is
+     * evicted or it is invalidated.
+     *
+     * `sizeBytes` is what `groundedReport` in `@cloudbitmaps/tools` prices storage from.
      *
      * ```ts
-     * const { generation, cardinality, metadata } = await store.segment('active-30d').stat();
+     * const { generation, cardinality, metadata, sizeBytes } = await store.segment('active-30d').stat();
      * ```
      */
     stat(): Promise<SegmentStat>;
@@ -1714,7 +1832,7 @@ export declare class Segment {
      * for await (const id of audience.everyNth(1_000)) ends.push(id); // the end of each window of 1,000
      * ```
      */
-    everyNth(n: number, options?: IdRange): AsyncIterable<number>;
+    everyNth(n: number, options?: IdRange): IdIterator;
     /**
      * The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
      * it was given ({@link excludesOf}) and mapped down to the plain refs
@@ -1741,7 +1859,6 @@ export declare class Segment {
      * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
      */
     intersect(others: Segment[], options?: CombineOptions): IdStream;
-    private intersectAs;
     /**
      * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
      * are superseded, not added to. Streaming + bounded-memory; the result is one immutable object published
@@ -1774,7 +1891,6 @@ export declare class Segment {
      * segment once (`unionInto`, or a load) is the cheaper shape.
      */
     union(others: Segment[], options?: CombineOptions): IdStream;
-    private unionAs;
     /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
     unionInto(dest: Segment, others: Segment[], options?: MaterializeOptions): Promise<MaterializeResult>;
     /**
@@ -1789,23 +1905,16 @@ export declare class Segment {
      * the suppression folds into the same pass rather than materializing an intermediate segment first.
      */
     andNot(excludes: Segment[], options?: BaseCombineOptions): IdStream;
-    private andNotAs;
     /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
     andNotInto(dest: Segment, excludes: Segment[], options?: AndNotIntoOptions): Promise<MaterializeResult>;
     /**
-     * Grounded cost report for this segment: storage cost from its **real** `.crbm` size (exact, no payload
-     * reads); request cost from the supplied `workload` rates. A segment with no Storage generation reports zero
-     * storage. The pointer refresh is priced at the store's own `cache.genTtlMs`, or at none when the store never
-     * refreshes, unless the workload sets `genTtlMs`. See {@link CostReport} — it always includes a verdict (incl.
-     * the lose-zone).
-     * @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
-     * needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
-     */
-    costReport(options?: {
-        pricing?: PricingProfile;
-        workload?: Workload;
-    }): Promise<CostReport>;
+     * The one body of the three combines, read as ids or as chunks. `operands` is `others` for `intersect` and
+     * `union`, and `excludes` for `andNot`, which takes no `exclude` option.
+     */
+    private combineAs;
+    /** The one body of the three `*Into` verbs, after each has checked its own options. */
+    private combineInto;
 }
 export { deserializePortable };
-export { MemoryStorage, LocalFsStorage, CrbmStorageChunkSource, destroySegment, eraseNamespace, excludingReservedRows, readRetentionPolicy, MIN_EXPIRES_AT_MS, LEASE_SKEW_MS, MAX_LEASE_MS, MAX_LEASES_PER_SEGMENT, InProcessKeystore, NodeAead, aadFor, CloudRoaringError, ValidationError, WriteConflictError, IntegrityError, NotFoundError, UnsupportedError, CapabilityError, TransientError, KeyUnavailableError, BudgetExceededError, LeaseExpiredError, LeaseLimitError, StaleOperandError, isCloudRoaringError, isWriteConflictError, isTransientError, isNotFoundError, isIntegrityError, isValidationError, isLeaseExpiredError, isLeaseLimitError, isStaleOperandError, isUnsupportedError, isCapabilityError, isBudgetExceededError, isKeyUnavailableError, CrbmReader, BufferReader, CountingMetricsSink, RecordingAuditSink, DEFAULT_RETRY_POLICY, AWS_US_EAST_1_ONDEMAND, ELASTICACHE_REDIS_US_EAST_1_ONDEMAND, ONE_REDIS_HA_CLUSTER, } from '@cloudbitmaps/core';
-export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CostReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, ClearRegistrySummary, DropResult, EncodedChunk, EraseDeps, EstimateInput, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GenerationMetadata, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PortableBitmap, PricingProfile, RedisNodeType, RedisSizing, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, RegistryWriteOptions, LeaseEntry, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SealedRegistrySummary, SegmentInfo, SegmentRef, SegmentSize, SegmentSizing, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, Token, Workload, WrappedDek, } from '@cloudbitmaps/core';
+export { MemoryStorage, LocalFsStorage, CrbmStorageChunkSource, destroySegment, eraseNamespace, excludingReservedRows, readRetentionPolicy, MIN_EXPIRES_AT_MS, LEASE_SKEW_MS, MAX_LEASE_MS, MAX_LEASES_PER_SEGMENT, InProcessKeystore, NodeAead, aadFor, CloudRoaringError, ValidationError, WriteConflictError, IntegrityError, NotFoundError, UnsupportedError, CapabilityError, TransientError, KeyUnavailableError, BudgetExceededError, LeaseExpiredError, LeaseLimitError, StaleOperandError, isCloudRoaringError, isWriteConflictError, isTransientError, isNotFoundError, isIntegrityError, isValidationError, isLeaseExpiredError, isLeaseLimitError, isStaleOperandError, isUnsupportedError, isCapabilityError, isBudgetExceededError, isKeyUnavailableError, CrbmReader, BufferReader, CountingMetricsSink, RecordingAuditSink, DEFAULT_RETRY_POLICY, } from '@cloudbitmaps/core';
+export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, ClearRegistrySummary, DropResult, EncodedChunk, EraseDeps, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportSkipped, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GenerationMetadata, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PortableBitmap, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, RegistryWriteOptions, KnownGeneration, LeaseEntry, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SealedRegistrySummary, SegmentInfo, SegmentRef, SegmentSize, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, StorageDeleteOptions, TailRead, Token, WrappedDek, } from '@cloudbitmaps/core';