@cloudbitmaps/core 0.20.0 → 0.22.0

package/dist/core/audit.d.ts

@@ -38,7 +38,7 @@ export type AuditEvent = {
 } | {
     /**
      * A segment's pointer was moved by a rollback to a generation still in the bucket that the caller named:
-     * **backwards**, or forward with `allowForward`, which undoes an earlier rollback.
+     * **backwards**, or forward onto a generation a rollback left above the pointer, which undoes an earlier rollback.
      *
      * The only non-forward-only pointer move in the library, and the only one no automatic path can perform —
      * a human decided the current generation was wrong and named the one they wanted. Every other pointer move
@@ -54,6 +54,14 @@ export type AuditEvent = {
     readonly incarnation?: string;
     readonly fromGeneration: number | null;
     readonly generation: number;
+    /**
+     * Present, and `true`, when the row did not vouch for the target: the call passed `allowUnknown` for a generation the
+     * row does not know or whose object is not the one it recorded, or the row has no usable record of its generations
+     * (a registry that drops the field, or a pointer moved by hand) or the target is below a full record's bound, where the
+     * pointer alone decides for a target below it. Absent when the move was
+     * checked against the row's record.
+     */
+    readonly unknown?: true;
 } | {
     /**
      * A load was **refused**: it did not publish, because its result failed a guard or another writer got there
@@ -109,7 +117,9 @@ export type AuditEvent = {
      * exists only for a finished erasure. A call that ends otherwise throws or reports `erased: false` with another
      * reason and emits nothing, even when it had deleted a holder; its re-run emits when it finishes, and reports
      * `'not-member'` with no event when another collection deleted the last holder meanwhile, as its ledger entry
-     * does.
+     * does. One finished erasure emits it and then throws `NotFoundError`: one that found, after its deletes, the
+     * segment's pointer on a generation it had deleted (a rollback moved it there meanwhile), so the deletion is on
+     * the record although the call cannot answer.
      *
      * `fromGeneration` is the newest generation the call found holding the id, and is absent when it found the id in
      * none and deleted only objects it could not search: then the ledger lists no entry for the segment, and this

package/dist/core/bit-route.d.ts

@@ -1,11 +1,18 @@
+/**
+ * BitRouteSplitter — maps a 32-bit id to its chunk (top 16 bits) and remainder
+ * (bottom 16 bits), per the data model.
+ */
+import { ValidationError } from './errors.js';
 export declare const U32_MAX = 4294967295;
 export declare const CHUNK_COUNT = 65536;
-export interface BitRoute {
+interface BitRoute {
     /** Top 16 bits — the chunk key (0..65535). */
     readonly chunkKey: number;
     /** Bottom 16 bits — the value within the chunk (0..65535). */
     readonly remainder: number;
 }
+/** Was `err` thrown by {@link splitId} for an id outside the u32 range? */
+export declare function isBadIdError(err: unknown): err is ValidationError;
 /** Split a u32 id into `{ chunkKey, remainder }`. Throws `ValidationError` for non-u32 input. */
 export declare function splitId(id: number): BitRoute;
 /**
@@ -19,3 +26,4 @@ export declare const MAX_REMAINDER: number;
  * invariant 5); `remainder` is masked to 16 bits.
  */
 export declare function joinId(chunkKey: number, remainder: number): number;
+export {};

package/dist/core/blob.d.ts

@@ -12,12 +12,23 @@ export interface BlobSink {
 export interface BlobReader {
     /** Exactly `length` bytes starting at `offset`. Out-of-range is a `ValidationError`, never a short read. */
     getRange(offset: number, length: number): Promise<Uint8Array>;
-    /** The last `min(maxBytes, size)` bytes plus the total size — the one-GET footer+index path. */
+    /**
+     * The last `min(maxBytes, size)` bytes plus the total size — the one-GET footer+index path. `maxBytes` is a safe
+     * integer, zero or less for an empty tail; any other length is refused with `ValidationError`.
+     */
     getTail(maxBytes: number): Promise<{
         bytes: Uint8Array;
         size: number;
     }>;
 }
+/**
+ * Check the length a tail read asks for, the one rule every tail read holds to (each storage driver's `getTail` and
+ * {@link BufferReader.getTail}): a safe integer. Zero or less answers an empty tail with the size, and a length past
+ * the object's size answers the whole object. Anything else is refused with `ValidationError` before anything is read:
+ * `NaN`, a fraction (a negative one too), an infinity of either sign, and a whole number past the safe range (2^53 or
+ * more, or -2^53 or less), so no reader takes one for some other length.
+ */
+export declare function checkTailLength(maxBytes: number): void;
 /** In-memory `BlobSink` that accumulates written bytes; `bytes()` returns the concatenation. */
 export declare class BufferSink implements BlobSink {
     private readonly chunks;

package/dist/core/codec.d.ts

@@ -103,7 +103,7 @@ export interface CodecBitmap {
     /**
      * This set as storage chunks: ascending by chunk key, empty chunks left out, each payload exactly the bytes that
      * {@link CodecInterface.fromValues} of that chunk's low 16 bits, then {@link optimize}, then {@link serialize}
-     * would give. So a generation written from these is byte for byte the one the same ids write. It may re-encode
+     * would give. So a generation written from these holds chunk for chunk what the same ids write. It may re-encode
      * this bitmap in place first, as `optimize()` does; membership is untouched.
      *
      * **Flavor-author surface, and optional.** A load from a bitmap or from serialized bytes writes these, never

package/dist/core/combine-expr.d.ts

@@ -1,10 +1,14 @@
-/** A string names an operand; a node holds exactly one operator over a non-empty list of expressions. */
-export type CombineExpr = string | {
-    and: CombineExpr[];
+/**
+ * An expression over the operands of a batch combine (a store's `materializeMany`): a string names an operand; a node
+ * holds exactly one of `and`, `or` or `andNot` over a non-empty list of expressions. `andNot` subtracts every entry after
+ * the first from the first, so it takes at least two. Plain JSON, so it can be logged and sent between tasks.
+ */
+export type Expr = string | {
+    and: Expr[];
 } | {
-    or: CombineExpr[];
+    or: Expr[];
 } | {
-    andNot: CombineExpr[];
+    andNot: Expr[];
 };
 /** The most operator nodes one expression may nest. A literal that contains itself is refused by it. */
 export declare const MAX_EXPR_DEPTH = 64;
@@ -21,7 +25,7 @@ export type ExprNode = {
     readonly kids: readonly ExprNode[];
 };
 /** How many nodes one {@link compileExpr} call may still allocate; shared across the expressions of one output. */
-export interface NodeBudget {
+interface NodeBudget {
     remaining: number;
 }
 /**
@@ -38,8 +42,6 @@ export interface KeyBounds {
     readonly keys: Uint16Array;
     readonly card: Uint32Array;
 }
-/** The keys of an operand the plan cannot know (a fed one): every key. Compared by identity, never by content. */
-export declare const ANY_KEYS: Uint16Array;
 /** The bounds of an operand the plan cannot know. */
 export declare const ANY_BOUNDS: KeyBounds;
 /** What {@link planKeys} keeps of each operator node: the keys it can be non-empty at. */

package/dist/core/combine-feed.d.ts

@@ -1,21 +1,14 @@
+import type { CodecBitmap, CodecInterface } from './codec.js';
 /**
- * The chunk-ordered feed of a batch combine: operands that arrive as records, one chunk key at a time, and are checked
- * and converted as they arrive. Pure: it reads through the iterable it is given and knows no store, clock or storage.
- *
- * ```
- * record { key, operands: { name: Uint32Array } }
- *   check    key in range and not below the last; every name declared; every value a real Uint32Array; ids strictly
- *            ascending and inside the key; each operand at most once per key        (a refusal, never a repair)
- *   convert  each operand's ids into one compressed bitmap, charged to the ledger as it is built
- *   group    the records of one key, held until a record with a higher key arrives or the feed ends
- * end of feed   counts equal what was seen; a declared operand that never appeared is refused unless allowed
- * ```
- *
- * Nothing here ever reads a bad feed as fewer members: every check is a refusal, and a refusal ends the feed.
+ * One record of the feed of a batch combine (a store's `materializeMany`): the ids some of the fed operands hold at one
+ * chunk key. `key` is `id >>> 16`, an integer from 0 to 65,535, never below the previous record's; a key may arrive as
+ * several consecutive records, each carrying some of the operands, and one record per key is the usual shape. `operands`
+ * maps a declared fed name to its ids at the key: a real `Uint32Array` (not an `Int32Array`, `Buffer`, array or
+ * look-alike), strictly ascending, every id inside the key. A name is given at most once per key, and an empty array is
+ * the same as leaving the name out. The record is copied as it arrives, so the caller may drop or reuse its arrays once
+ * it has yielded it.
  */
-import type { CodecBitmap, CodecInterface } from './codec.js';
-/** One record of a feed: the ids each named operand holds at one chunk key. */
-export interface CombineManyFeedRecord {
+export interface MaterializeManyFeedRecord {
     /** The chunk key, `id >>> 16`, from 0 to 65,535. Non-decreasing over the feed; a key may arrive as several records. */
     readonly key: number;
     /** Ids by operand name: ascending, unique, every one inside `key`. An operand is named at most once per key. */
@@ -26,7 +19,7 @@ export interface CombineManyFeed {
     /** The fed operand names, declared up front. */
     readonly names: readonly string[];
     /** The records, read once, in key order. */
-    readonly records: AsyncIterable<CombineManyFeedRecord>;
+    readonly records: AsyncIterable<MaterializeManyFeedRecord>;
     /**
      * The ids each fed operand holds over the whole feed, keyed by exactly `names`: an object, or a function called once
      * after the last record. A count that differs from what the feed held refuses every fed output.
@@ -56,7 +49,7 @@ export declare class FeedClosed extends Error {
 export declare class FeedStale extends Error {
 }
 /** What a {@link FeedCursor} needs from the pass that runs it. */
-export interface FeedCursorDeps {
+interface FeedCursorDeps {
     readonly feed: CheckedFeed;
     readonly codec: CodecInterface;
     /** The index the first fed operand has among the call's operands. */
@@ -136,3 +129,4 @@ export declare class FeedCursor {
     /** The counts and the empty rule, run once, after the iterator has ended and before anything is published. */
     private finish;
 }
+export {};

package/dist/core/combine-many.d.ts

@@ -1,18 +1,16 @@
 import type { CodecBitmap, CodecInterface } from './codec.js';
 import type { IMetricsSink } from './metrics.js';
 import type { CombineManyFeed } from './combine-feed.js';
-import type { CombineExpr } from './combine-expr.js';
+import type { Expr } from './combine-expr.js';
 import type { Clock } from './determinism.js';
 import type { Budget } from './budget.js';
 import type { LoadGuard } from './load.js';
 import type { GenerationMetadata, SegmentRef, StorageChunkSource } from './ports.js';
-export type { CombineExpr } from './combine-expr.js';
+export type { Expr } from './combine-expr.js';
 export { MAX_EXPR_DEPTH, MAX_EXPR_NODES } from './combine-expr.js';
-export type { CombineManyFeed, CombineManyFeedRecord } from './combine-feed.js';
+export type { CombineManyFeed, MaterializeManyFeedRecord } from './combine-feed.js';
 /** The range requests a pass keeps in flight at once across all its operands, inside a client's socket pool. */
 export declare const MAX_RANGES_IN_FLIGHT = 64;
-/** The largest `publishConcurrency` and `concurrency` a call may ask for. */
-export declare const MAX_BATCH_PARALLELISM = 1024;
 /** One named stored operand, as the pass reads it. */
 export interface CombineManyOperand {
     readonly name: string;
@@ -50,9 +48,9 @@ export interface CombineManyOperand {
 }
 /** What one output asks for. */
 export interface CombineManyOutput<R> {
-    readonly expr: CombineExpr;
+    readonly expr: Expr;
     /** Subtracted from the result: a root `andNot`. */
-    readonly exclude?: readonly CombineExpr[];
+    readonly exclude?: readonly Expr[];
     readonly allowEmpty?: boolean;
     readonly guard?: LoadGuard;
     readonly metadata?: GenerationMetadata;
@@ -128,8 +126,8 @@ export type CombineManyOutcome<R> = {
     readonly ok: false;
     readonly error: unknown;
 };
-/** What the call did to one operand. */
-export interface CombineManyOperandStats {
+/** What a batch combine (a store's `materializeMany`) did to one operand: see {@link MaterializeManyStats.operands}. */
+export interface MaterializeManyOperandStats {
     /** Whether an output of the call read it. An operand no output names is not read. */
     readonly read: boolean;
     /** Whether the operand was read at a pinned generation. */
@@ -157,8 +155,8 @@ export interface CombineManyOperandStats {
     /** Chunk reads made of it, over all groups. */
     readonly chunkReads: number;
 }
-/** What the call did to one output. */
-export interface CombineManyOutputStats {
+/** What a batch combine (a store's `materializeMany`) did to one output: see {@link MaterializeManyStats.outputs}. */
+export interface MaterializeManyOutputStats {
     /** The names of the operands its expression reads: the stored ones in call order, then the fed ones in `feed.names` order. */
     readonly operands: readonly string[];
     /** The group that computed it (from 0), or `null` when it never ran. */
@@ -173,7 +171,11 @@ export interface CombineManyOutputStats {
     /** The most its buffer held resident, in the ledger's own measure; compare with `boundBytes` to see how tight a plan was. */
     readonly peakBytes: number;
 }
-export interface CombineManyStats {
+/**
+ * What a batch combine (a store's `materializeMany`) did: the groups it ran, its requests by class, the chunk-read budget
+ * it used, the most it held resident, and, for each operand, the generation it was pinned at, began at and ended at.
+ */
+export interface MaterializeManyStats {
     /** Groups run, the re-runs of outputs a lying index made too big included. */
     readonly groups: number;
     readonly requests: {
@@ -235,19 +237,17 @@ export interface CombineManyStats {
         readonly ids: number;
     };
     /** By operand name, an object with no prototype, so any name is a key. */
-    readonly operands: Readonly<Record<string, CombineManyOperandStats>>;
-    readonly outputs: readonly CombineManyOutputStats[];
+    readonly operands: Readonly<Record<string, MaterializeManyOperandStats>>;
+    readonly outputs: readonly MaterializeManyOutputStats[];
 }
 export interface CombineManyRun<R> {
     readonly outputs: readonly CombineManyOutcome<R>[];
-    readonly stats: CombineManyStats;
+    readonly stats: MaterializeManyStats;
 }
 /** A request that passed every check that needs no I/O: what {@link runCombineMany} runs. */
 export interface CompiledCombineMany<R> {
     readonly request: CombineManyRequest<R>;
 }
-/** The most nodes the expressions of one call may hold in all, so that a thousand large outputs cannot be planned at once. */
-export declare const MAX_CALL_NODES = 200000;
 /**
  * Check a request: every expression, option and number, before any request is made. Throws {@link ValidationError}
  * naming the output index and the path. Pure.
@@ -259,7 +259,7 @@ export declare function compileCombineMany<R>(request: CombineManyRequest<R>): C
  */
 export declare function rebindCombineMany<R>(compiled: CompiledCombineMany<R>, operands: readonly CombineManyOperand[]): CompiledCombineMany<R>;
 /** What {@link GroupCost} reads of an output. */
-export interface GroupCostItem {
+interface GroupCostItem {
     readonly bound: number;
     readonly serializedBound: number;
     readonly rootKeys: Uint16Array;
@@ -270,14 +270,15 @@ export interface GroupCostItem {
 }
 /**
  * Work counters of the planner, for the tests that hold it to its cost: how many times a group was priced, how many
- * operands those prices walked, and how many group plans were built. Plain numbers, never read by the planner.
+ * operands those prices walked, and how many group plans were built. A call counts into the one it is given
+ * ({@link runCombineManyCounted}), and into none otherwise; the planner never reads them.
  */
-export declare const combineWork: {
+export interface PlanWork {
     adds: number;
     withs: number;
     operandSteps: number;
     planBuilds: number;
-};
+}
 /**
  * The resident bytes a group needs, kept as outputs are added so that one is priced in work proportional to its own
  * operands, not to the group's size: its buffered outputs, its operand streams, the chunks being evaluated, the objects
@@ -286,6 +287,8 @@ export declare const combineWork: {
 export declare class GroupCost {
     private readonly streamBytes;
     private readonly publishConcurrency;
+    /** Where this pricing is counted, when it is. */
+    private readonly work?;
     private outputs;
     private depth;
     /** The serialized bounds of the largest outputs, descending: at most `publishConcurrency` of them are written at once. */
@@ -294,7 +297,9 @@ export declare class GroupCost {
     private streams;
     private planWork;
     private readonly operands;
-    constructor(streamBytes: (operand: number) => number, publishConcurrency: number);
+    constructor(streamBytes: (operand: number) => number, publishConcurrency: number, 
+    /** Where this pricing is counted, when it is. */
+    work?: PlanWork | undefined);
     /** The cost of the group with `o` added. */
     with(o: GroupCostItem): number;
     add(o: GroupCostItem): void;
@@ -313,6 +318,8 @@ export declare class GroupCost {
  * budget, both before any chunk is read.
  */
 export declare function runCombineMany<R>(compiled: CompiledCombineMany<R>, deps: CombineManyDeps): Promise<CombineManyRun<R>>;
+/** {@link runCombineMany}, counting the planner's work into `work`. Not exported from the package. */
+export declare function runCombineManyCounted<R>(compiled: CompiledCombineMany<R>, deps: CombineManyDeps, work: PlanWork): Promise<CombineManyRun<R>>;
 /**
  * The `pointerId` of a row's version (`<generation>:<pointerId>`, as a pin holds it), or `undefined` where the version
  * names no row. A live version from the `.crbm` source ends with the opened object's fingerprint: read it through

package/dist/core/concurrency.d.ts

@@ -13,3 +13,11 @@
  * S3 page, chunk map), so an array keeps the pool trivial and race-free (no shared-iterator hazard).
  */
 export declare function mapWithConcurrency<T, R>(items: readonly T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
+/**
+ * `Promise.all` over `items` with at most `limit` calls of `fn` in flight: the results in input order, or the first
+ * failure, at once. Unlike {@link mapWithConcurrency} it does not wait for the calls still in flight before it rejects,
+ * as `Promise.all` does not; it starts no call after a failure, and what the calls in flight answer is dropped. For reads
+ * a caller abandons on the first fault, never for work that must finish or be accounted for once begun. When every item
+ * fits under `limit` it is `Promise.all` itself, so a small fan-out costs and interleaves exactly as `Promise.all` does.
+ */
+export declare function boundedAll<T, R>(items: readonly T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;

package/dist/core/consistency.d.ts

@@ -3,18 +3,23 @@
  * generations can be restored **independently**, so a failover can recover the registry *ahead of* the storage
  * objects — leaving `currentGen` pointing at a generation whose `.crbm` isn't present yet. That is likelier
  * than it sounds even with both in one bucket: a restore scoped to a prefix, or replayed
- * per-object from a version history, recovers the two prefixes at different points. That's a torn restore: reads of the affected segment then throw. This scan
+ * per-object from a version history, recovers the two prefixes at different points. That's a torn restore: reads of
+ * the affected segment then throw. This scan
  * detects it up front (run it at startup after a restore) instead of discovering it on the first read.
  *
  * Read-only; bounded fan-out. `destroyed` (crypto-shredded) segments are skipped — their Storage is intentionally
  * gone/unreadable, not a torn restore. A segment whose Storage/registry can't be read this pass is recorded in
- * `errored` (never aborts the scan). Each segment is checked against its **authoritative live pointer** — one
+ * `errored` (never aborts the scan). One segment can add several entries to `inconsistent` (its current object, and with
+ * `generations` one per known generation whose object is not the one recorded) and can also be in `errored`, when a
+ * read failed after those were found; with `generations` a segment costs up to one read per entry of its ledger.
+ * Each segment is checked against its **authoritative live pointer** — one
  * strong `registry.get` per segment — never the enumeration snapshot from `registry.list`, which a driver may
  * serve eventually-consistently and so lag a recent in-place pointer advance: trusting it would both
  * miss a torn *live* generation and cry torn on a generation the pointer has already advanced past (GC'd during
- * the scan). Residual: a load's publish plus a GC landing in the tiny per-segment get→list gap can still yield a
- * transient false positive — run the scan against a quiesced fleet (the documented restore procedure), or re-run
- * to confirm a reported tear.
+ * the scan). Residual: a load's publish plus a GC landing in the per-segment gap between the pointer read and the
+ * object reads (one read, or with `generations` as many as the ledger names) can still yield a transient false
+ * positive — run the scan against a quiesced fleet (the documented restore procedure), or re-run to confirm a
+ * reported tear.
  */
 import type { IStorageDriver, IRegistryDriver } from './ports.js';
 import type { IKeystore } from './crypto.js';
@@ -33,7 +38,11 @@ export { DEFAULT_MAX_SCAN_SEGMENTS } from './registry-scan.js';
 export interface ConsistencyIssue {
     readonly segment: string;
     readonly namespace?: string;
-    /** The registry's `currentGen` for the segment — the generation whose `.crbm` is missing from Storage. */
+    /**
+     * The registry's `currentGen` for the segment. For `missing-storage-generation` it is the generation whose `.crbm` is
+     * missing from Storage; for the other issues it is the segment's current generation, whichever generation the issue
+     * is about (see `generation`).
+     */
     readonly currentGen: number;
     /**
      * `missing-storage-generation`: `currentGen` references a Storage generation that is not present (torn restore).
@@ -44,8 +53,20 @@ export interface ConsistencyIssue {
      * that opens the object refuses it; one whose count or metadata disagrees is answered from until a read opens the
      * object, which stops that store using it; one whose fingerprint names no object is not used, so every read opens the
      * object unchecked.
+     * `known-generation-mismatch` (only when asked for): the object under the number `generation` is not the one the row's
+     * ledger of known generations recorded for it: its footer checksum differs, or its footer fails its own checks (it is
+     * not a `.crbm` object, or is cut short). It says nothing of what a rollback does with that generation.
+     * What it covers: each object's footer is held to the recorded checksum, so another object, or bytes that are not a
+     * `.crbm`, under a known number is found. It does not find damage inside an object whose footer is intact (a cut or a
+     * flipped byte in the body: a read of that generation reports it as `IntegrityError`), nor a metadata change that keeps
+     * the footer's bytes.
      */
-    readonly issue: 'missing-storage-generation' | 'summary-mismatch';
+    readonly issue: 'missing-storage-generation' | 'summary-mismatch' | 'known-generation-mismatch';
+    /**
+     * With `known-generation-mismatch` only: the generation whose object is not the one the row recorded. It is
+     * `currentGen` when the current object is the one that differs. Absent on every other issue.
+     */
+    readonly generation?: number;
 }
 /** A segment that could not be checked this pass (Storage/registry read fault) — not proof of a torn restore. */
 export interface ConsistencyErrorEntry {
@@ -57,12 +78,17 @@ export interface ConsistencyErrorEntry {
 export interface ConsistencyReport {
     /** Registered segments scanned (a `destroyed` segment counts as scanned but is never an issue). */
     readonly checked: number;
-    /** Segments whose `currentGen` `.crbm` is absent — recover the object store (or restore to a coherent point). */
+    /**
+     * The issues found, one entry each: a segment whose `currentGen` `.crbm` is absent (recover the object store, or
+     * restore to a coherent point), and with the options asked for the other kinds. One segment can have several entries:
+     * at most one for its current object plus one per entry of its ledger, so at most 1 plus the ledger's length (384 at
+     * most). The ledger's length is bounded by the registry, and the report is not capped.
+     */
     readonly inconsistent: ConsistencyIssue[];
     /**
      * Segments that couldn't be read this pass (a transient/partial object store during a restore is exactly when
-     * this runs) — **triage these too**: an unread segment is neither proven-coherent nor proven-torn. Empty on a
-     * clean pass.
+     * this runs) — **triage these too**: an unread segment is neither proven-coherent nor proven-torn. A segment can be in
+     * this list and in `inconsistent` too: what was found before the read that failed is kept. Empty on a clean pass.
      */
     readonly errored: ConsistencyErrorEntry[];
     /**
@@ -71,19 +97,44 @@ export interface ConsistencyReport {
      * only when summaries were asked for.
      */
     readonly summariesUnchecked?: number;
+    /**
+     * With `generations: true`, the rows with a pointer whose ledger of known generations could not be used (absent, or
+     * failing the validation every reader of it applies, or not shaped as a ledger a registry of this library stores):
+     * none of their known generations was held against its object. A count, not an issue. Present only when generations
+     * were asked for.
+     */
+    readonly generationsUnchecked?: number;
 }
 /**
  * Verify every registered segment's `currentGen` `.crbm` actually exists in Storage. Enumerates the registry
- * (optionally one namespace) and, for each non-`destroyed` segment, checks the object store lists that
- * generation. Returns the torn segments in `inconsistent` (empty ⇒ coherent) and any unreadable segments in
+ * (optionally one namespace) and, for each non-`destroyed` segment, asks the object store for that generation with a
+ * zero-length tail read. Returns the torn segments in `inconsistent` (empty ⇒ coherent) and any unreadable segments in
  * `errored`.
  *
+ * When that read fails, in any way, the segment's listing decides and its own error is the segment's, so a missing
+ * object, a missing bucket and a failing store are reported as when every segment was listed. One difference: a store
+ * that answers reads but refuses listings (an identity without list permission, or a listing-only outage) reports each
+ * object that is there as present, where every segment was reported errored.
+ *
  * With `summaries: true` it also opens each segment's current object (one tail read, and a second for an index longer
- * than it) and holds the row's summary against it: the object it names by its fingerprint, and the count and metadata it
- * says, reporting `summary-mismatch` where they disagree, and for a clear summary whose fingerprint names no object,
- * which no read uses. A sealed summary
- * needs `deps.keystore`; without it the segment is counted in `summariesUnchecked`. Off by default, since the
- * default check only lists.
+ * than it) and holds the row's summary against it: the object it names by its fingerprint, and the count and metadata
+ * it says, reporting `summary-mismatch` where they disagree, and for a clear summary whose fingerprint names no
+ * object, which no read uses. For a row with a clear summary that open is the only request of the object; if it fails
+ * before its first read answered, the listing decides, and once that read has answered a failure is an error of the
+ * segment. A sealed summary needs `deps.keystore`; without it the segment is counted in `summariesUnchecked`. Off by
+ * default, since the default check only looks for the object.
+ *
+ * With `generations: true` it also holds every generation a row's ledger of known generations names, the current one
+ * included, against its object: one footer read each (104 bytes, no key, not retried), reporting
+ * `known-generation-mismatch` with the `generation` where the object's footer checksum is not the one the ledger records
+ * or its footer fails its own checks. That finds another object, or bytes that are not a `.crbm`, under a known number;
+ * it does not find damage inside an object whose footer is intact (a read of it reports that as `IntegrityError`), nor a
+ * metadata change that keeps the footer's bytes. A segment adds at most 1 plus its ledger's length issues. A named
+ * generation that is not in the bucket is not reported. A row with a pointer whose ledger cannot be used (absent, failing
+ * the validation every reader of it applies, or not shaped as a ledger a registry of this library stores) is counted in
+ * `generationsUnchecked` and none of its objects is read. The listing is not used: a read of a missing object
+ * answers `NotFoundError`, which is how an absent one is told. A segment whose current object is missing is reported as
+ * ever and its older generations are still checked. Off by default.
  */
 export declare function runConsistencyCheck(deps: {
     readonly storage: IStorageDriver;
@@ -94,4 +145,5 @@ export declare function runConsistencyCheck(deps: {
     concurrency?: number;
     maxScanSegments?: number;
     summaries?: boolean;
+    generations?: boolean;
 }): Promise<ConsistencyReport>;

package/dist/core/crbm/reader.d.ts

@@ -7,7 +7,7 @@ import type { ChunkRead, GenerationMetadata, ReadChunksOptions } from '../ports.
  * `cardinalityMinusOne` stores `cardinality - 1` because a chunk's cardinality runs `[1, 65536]` and 65,536 does
  * not fit a `u16`; an offset is a `f64` because an object past 4 GiB puts a payload beyond a `u32`.
  */
-export interface ParsedIndex {
+interface ParsedIndex {
     readonly keys: Uint16Array;
     readonly cardinalityMinusOne: Uint16Array;
     readonly lengths: Uint32Array;
@@ -29,7 +29,12 @@ export declare const RETAINED_BYTES_PER_INDEX_ENTRY = 20;
  */
 export declare function indexCapacity(indexLength: number): number;
 export interface CrbmReaderOptions {
-    /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
+    /**
+     * How many bytes the first tail read asks for, a whole number (default 256 KiB; clamped up to at least the footer
+     * size). Any value of at least an object's `tailBytes` opens it in that one read. Where an open keeps a small object's
+     * chunks (a `CrbmStorageChunkSource`'s does), it keeps them only when this read covers the whole object, so a value
+     * below the default can leave them to reads of their own.
+     */
     readonly tailBytes?: number;
     /** Hard cap on a single chunk payload length (default: the 1 MiB decode cap, plus the 28-byte nonce and tag on an encrypted object). The engine decodes under `maxBitmapBytes`; raise this with it. */
     readonly maxPayloadBytes?: number;
@@ -49,6 +54,20 @@ export interface CrbmReaderOptions {
      */
     readonly lineage?: unknown;
 }
+/**
+ * Refuses a `size` that is not a whole byte count its own tail fits in. The size is the tier's word too: one that
+ * is not would turn off every bounds check an open makes against it.
+ */
+export declare function checkSize(tail: Uint8Array, size: number): void;
+/**
+ * What the footer of the object behind `blob` says with no key and no index, from one tail read of a footer's worth:
+ * whether it is encrypted, and its CRC (the one in {@link CrbmReader.fingerprint}). A footer that fails its own checks
+ * throws, as an open would.
+ */
+export declare function readFooter(blob: BlobReader): Promise<{
+    readonly encrypted: boolean;
+    readonly footerCrc: number;
+}>;
 /**
  * Whether the object behind `blob` says it is encrypted, from one tail read of a footer's worth, with no key and no
  * index; a footer that fails its own checks throws, as an open would. For a caller that must not open an object with
@@ -61,6 +80,14 @@ export declare function footerSaysEncrypted(blob: BlobReader): Promise<boolean>;
  * object and that region is at most `keepChunkBytesUpTo` bytes. Not exported from a package entry.
  */
 export declare function openCrbmReaderKeeping(blob: BlobReader, options: CrbmReaderOptions, keepChunkBytesUpTo: number | undefined): Promise<CrbmReader>;
+/**
+ * How many bytes the first tail read of an open asks for: the caller's `tailBytes` or the default, and a footer's worth
+ * at least. Not exported from a package entry: {@link CrbmReader.open} and a caller that sends that read ahead of it
+ * ask for the same size.
+ */
+export declare function openTailBytes(options: {
+    readonly tailBytes?: number;
+}): number;
 export declare class CrbmReader {
     private readonly blob;
     private readonly objectSize;
@@ -102,6 +129,11 @@ export declare class CrbmReader {
     get fingerprint(): string;
     /** Total object bytes (from the one-GET tail read): the `sizeBytes` a segment's `stat()` reports. */
     get sizeBytes(): number;
+    /**
+     * The bytes of the object after its chunk payloads: any extension block, the index and the footer. A tail read of
+     * this many bytes covers all of them.
+     */
+    get tailBytes(): number;
     /**
      * Retained JS heap of this reader's parsed index — the weight the storage reader cache bounds on (a wide
      * segment's parsed index, not its payloads, dominates the reader's footprint): the exact byte length of the
@@ -134,6 +166,16 @@ export declare class CrbmReader {
     count(): number;
     /** The slot holding `chunkKey` (binary search over the ascending keys), or -1 when this generation has none. */
     private slotOf;
+    /**
+     * {@link slotOf} for a key at or after the one that held slot `at`, which a walk over ascending keys has: the slot
+     * itself or the next are tried first, so a read of nearly every chunk costs no search.
+     */
+    private slotFrom;
+    /**
+     * The extents of the chunks `chunkKeys` name that this generation holds, in key order, each once (a key given twice
+     * is read once). A walk, so nothing is looked up until the plan pulls it.
+     */
+    private neededExtents;
     has(chunkKey: number): boolean;
     /**
      * Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
@@ -168,10 +210,13 @@ export declare class CrbmReader {
      * as {@link getChunk} answers for an absent key. A key given twice is read once and answered at both positions with
      * the same view.
      *
-     * The whole key list is planned once, up front, against this one generation's index. The ranges then run through a
-     * window of `options.concurrency` reads ({@link MAX_RANGES_IN_FLIGHT} by default) that counts a range until the
-     * consumer has taken it, so a stream never holds more than that many ranges, in flight or landed, however many keys
-     * it was given or how slowly it is read; with `options.ramp` the window opens 1, 2, 4 … wide. A range is at most 1 MiB
+     * The key list is planned against this one generation's index ({@link planChunkReads}), as a walk the window pulls
+     * from only as it opens a range, so a stream that stops early has planned little: the same ranges, in the same
+     * order, as a plan of the whole list would make, and each chunk's extent is checked before the range that carries
+     * it is requested. The ranges run through a window of `options.concurrency` reads ({@link MAX_RANGES_IN_FLIGHT} by
+     * default) that counts a range until the consumer has taken it, so a stream never holds more than that many ranges,
+     * in flight or landed, however many keys it was given or how slowly it is read; with `options.ramp` the window opens
+     * 1, 2, 4 … wide. A range is at most 1 MiB
      * unless it is one chunk, which is at most the payload cap, so the bound is that many times the per-chunk cap
      * (1 MiB, and 28 bytes more for an encrypted chunk). Nothing is read until the first item is asked for, and a
      * consumer that stops launches no more: ranges already in flight finish and are dropped, and a failure of one of
@@ -223,3 +268,4 @@ export type ChunkStream = AsyncGenerator<Omit<ChunkRead, 'version'>> & {
  */
 export declare function parseExtension(sections: Uint8Array, crypto: CrbmCrypto | undefined): GenerationMetadata | undefined;
 export declare function parseIndex(indexBytes: Uint8Array, payloadEnd: number, maxPayloadBytes: number, minPayloadBytes?: number): ParsedIndex;
+export {};