@cloudbitmaps/core 0.9.0 → 0.11.0

package/dist/core/consistency.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Fail-safe cross-store disaster-recovery check. The registry (`currentGen`) and the immutable `.crbm`
+ * generations can be restored **independently**, so a failover can recover the registry *ahead of* the storage
+ * objects — leaving `currentGen` pointing at a generation whose `.crbm` isn't present yet. That is likelier
+ * than it sounds even with both in one bucket: a restore scoped to a prefix, or replayed
+ * per-object from a version history, recovers the two prefixes at different points. That's a torn restore: reads of the affected segment then throw. This scan
+ * detects it up front (run it at startup after a restore) instead of discovering it on the first read.
+ *
+ * Read-only; bounded fan-out. `destroyed` (crypto-shredded) segments are skipped — their Storage is intentionally
+ * gone/unreadable, not a torn restore. A segment whose Storage/registry can't be read this pass is recorded in
+ * `errored` (never aborts the scan). Each segment is checked against its **authoritative live pointer** — one
+ * strong `registry.get` per segment — never the enumeration snapshot from `registry.list`, which a driver may
+ * serve eventually-consistently and so lag a recent in-place pointer advance: trusting it would both
+ * miss a torn *live* generation and cry torn on a generation the pointer has already advanced past (GC'd during
+ * the scan). Residual: a load's publish plus a GC landing in the tiny per-segment get→list gap can still yield a
+ * transient false positive — run the scan against a quiesced fleet (the documented restore procedure), or re-run
+ * to confirm a reported tear.
+ */
+import type { IStorageDriver, IRegistryDriver } from './ports.js';
+/** Default in-flight fan-out for the consistency scan — bounded, no thundering herd. */
+/**
+ * Default ceiling on how many registry records one consistency check may hold resident: 250,000.
+ *
+ * **Named `maxScanSegments`, not `maxSegments`, on purpose.** A `maxSegments`-style option caps how much work
+ * one cycle does and then *continues normally* (the retention sweep's `limit` is that shape). This one caps how
+ * much a scan may hold and *refuses* past it. Same-sounding names with opposite behaviour at the limit is a
+ * trap; the name says what it is — a ceiling on a scan that fails closed.
+ *
+ * Generous — fleets of 100K+ segments are the design target — while still bounding a DR drill's memory to
+ * something a modest operator box survives. Raisable, because a ceiling you cannot lift is a landmine.
+ */
+export { DEFAULT_MAX_SCAN_SEGMENTS } from './registry-scan.js';
+export interface ConsistencyIssue {
+    readonly segment: string;
+    readonly namespace?: string;
+    /** The registry's `currentGen` for the segment — the generation whose `.crbm` is missing from Storage. */
+    readonly currentGen: number;
+    /** The only issue class today: `currentGen` references a Storage generation that is not present (torn restore). */
+    readonly issue: 'missing-storage-generation';
+}
+/** A segment that could not be checked this pass (Storage/registry read fault) — not proof of a torn restore. */
+export interface ConsistencyErrorEntry {
+    readonly segment: string;
+    readonly namespace?: string;
+    /** The read error's message (never the raw error, so the report stays serializable/loggable). */
+    readonly error: string;
+}
+export interface ConsistencyReport {
+    /** Registered segments scanned (a `destroyed` segment counts as scanned but is never an issue). */
+    readonly checked: number;
+    /** Segments whose `currentGen` `.crbm` is absent — recover the object store (or restore to a coherent point). */
+    readonly inconsistent: ConsistencyIssue[];
+    /**
+     * Segments that couldn't be read this pass (a transient/partial object store during a restore is exactly when
+     * this runs) — **triage these too**: an unread segment is neither proven-coherent nor proven-torn. Empty on a
+     * clean pass.
+     */
+    readonly errored: ConsistencyErrorEntry[];
+}
+/**
+ * Verify every registered segment's `currentGen` `.crbm` actually exists in Storage. Enumerates the registry
+ * (optionally one namespace) and, for each non-`destroyed` segment, checks the object store lists that
+ * generation. Returns the torn segments in `inconsistent` (empty ⇒ coherent) and any unreadable segments in
+ * `errored`.
+ */
+export declare function runConsistencyCheck(deps: {
+    readonly storage: IStorageDriver;
+    readonly registry: IRegistryDriver;
+}, options?: {
+    namespace?: string;
+    concurrency?: number;
+    maxScanSegments?: number;
+}): Promise<ConsistencyReport>;

package/dist/core/cooperative.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Cooperative yielding for the long CPU-bound loops.
+ *
+ * Everything this library does on the hot read path is small. Bulk-load is the exception: it is the one place
+ * where a single call legitimately occupies the CPU for hundreds of milliseconds, and Node has exactly one thread
+ * to occupy. Measured on a 1M-id load of 65,536 chunks onto in-memory storage, on a loaded Apple M3 Pro: **518 ms
+ * wall, and 519 ms during which the event loop did not turn at all** (medians of 20 fresh-process runs). Wire that to a request handler and every other in-flight request on the instance — health
+ * checks included — waits out the whole load.
+ *
+ * The fix is not to make the work asynchronous. That was measured too, and it is a 7x *regression*: handing each
+ * chunk's insert to the threadpool costs ~9 µs of dispatch against ~1.5 µs of actual work once ids are spread
+ * across ~62,000 chunks (636 ms versus 92 ms). The work is genuinely CPU-bound and genuinely small per chunk;
+ * the only thing wrong with it is that it never stops to let anyone else go. So the fix is to keep the work
+ * synchronous and periodically hand the loop back.
+ *
+ * TWO TRAPS, both of which make a plausible implementation measure as no change at all:
+ *
+ * 1. **`await Promise.resolve()` — and equally `Clock.sleep(0)` — do nothing.** Both resolve on a microtask, and
+ *    the microtask queue drains to empty before the event loop advances a single phase. A loop awaiting them
+ *    yields to other *microtasks* and to nothing else; no pending I/O callback runs. Measured: 555 ms of
+ *    starvation against a 568 ms unyielded baseline **on the synthetic yield-primitive benchmark below** —
+ *    a different, smaller workload than the 442 ms end-to-end load above, which is why the two baselines
+ *    differ. The yield must be a **macrotask** — hence
+ *    `Clock.yieldNow`, backed by `setImmediate` in production wiring.
+ * 2. **Yielding per unit of work costs more than the work.** The yield has to be periodic, and `everyN` has to be
+ *    large enough that its cost disappears into the batch it interrupts.
+ *
+ * At `everyN = 1024` over that same synthetic workload the trade is close to free: 569 ms wall against a 568 ms
+ * baseline, with the worst event-loop gap down from 568 ms to 13.8 ms.
+ *
+ * **Three experiments, three sets of numbers — do not mix them.** (a) The end-to-end 1M-id bulk load:
+ * 518 ms wall / 519 ms blocked unyielded, 526 ms / 24 ms yielded. (b) This synthetic yield-primitive comparison:
+ * 568 ms / 568 ms unyielded, 569 ms / 13.8 ms with `setImmediate`. (c) A per-chunk insert microbenchmark:
+ * 92 ms sync versus 636 ms via the threadpool. Quoting a baseline from one against a result from another describes
+ * work none of them did, such as a 92 ms operation with 450 ms of starvation inside it.
+ */
+/**
+ * The only part of a `Clock` that yielding actually needs.
+ *
+ * Deliberately looser than `Clock` so a write path can opt in with whatever it has: a bulk load or a rewrite
+ * yields when its caller supplies a clock that can, and simply runs unbroken when handed nothing (or a bare
+ * `{ now }`). Requiring a full `Clock` would make every out-of-process caller wire one just to write a
+ * generation.
+ */
+export interface Yielder {
+    yieldNow?(): Promise<void>;
+    sleep?(ms: number): Promise<void>;
+}
+/**
+ * How many units of work pass between yields. Sized so the yield is unmeasurable against the batch it interrupts
+ * (~60 yields across a 1M-id load) while still capping any single blocking stretch in the low tens of ms.
+ */
+export declare const YIELD_EVERY = 1024;
+/**
+ * Build a periodic yield.
+ *
+ * Call the returned function **once per unit of work**. It returns `null` on the vast majority of calls — nothing
+ * to await, nothing allocated — and a promise only on the `everyN`th, so the caller pays for a yield only when
+ * one actually happens:
+ *
+ * ```ts
+ * const tick = yieldEvery(clock);
+ * for (const item of millions) {
+ *   work(item);
+ *   const pause = tick();
+ *   if (pause !== null) await pause;
+ * }
+ * ```
+ *
+ * The two-line shape is deliberate. `await tick()` would read better and would force a microtask turn on every
+ * single iteration — an unconditional cost on the very loop this exists to keep fast.
+ *
+ * With no `clock` the result always returns `null`: the loop runs unbroken, so adding a yield point to a code
+ * path can never change behaviour for a caller who did not opt in. `core/` cannot supply a clock itself — it is
+ * timer-free by lint, which is the whole reason scheduling goes through the seam — so `@cloudbitmaps/roaring`
+ * pre-binds a real one and flavor users get this with no wiring at all.
+ */
+export declare function yieldEvery(clock: Yielder | undefined, everyN?: number): () => Promise<void> | null;

package/dist/core/cost.d.ts

@@ -0,0 +1,254 @@
+/** Pluggable rate card. Rates differ by cloud/region/term and drift over time; the formulas don't. */
+export interface PricingProfile {
+    readonly name: string;
+    readonly storage: {
+        /** Object GET (per **million** requests). A ranged GET bills as a full GET. */
+        readonly getPerMillion: number;
+        /** Object PUT (per million) — what a load pays, per request. */
+        readonly putPerMillion: number;
+        readonly storagePerGiBMonth: number;
+        /**
+         * Requests one read costs when it needs the object's size first: a pointer read, and a segment's tail read.
+         * Default **1**, S3's, whose suffix-range GET returns the size with the bytes. **2** on GCS and Azure Blob,
+         * which read the metadata and then the bytes. A chunk read knows its range, and is one request everywhere.
+         */
+        readonly requestsPerSizedRead?: number;
+    };
+    /**
+     * The always-on Redis the verdict compares against: exactly one of the two. `sizedToData`, the default's, prices
+     * the cheapest cluster that holds the report's stored bytes; `monthlyUSD` is one cluster you name, whatever the
+     * data size. A profile carrying both is refused rather than read one way.
+     */
+    readonly redis: {
+        readonly monthlyUSD: number;
+        readonly sizedToData?: never;
+    } | {
+        readonly sizedToData: RedisSizing;
+        readonly monthlyUSD?: never;
+    };
+}
+/** One node type a Redis cluster can be built from, as its cloud prices it. */
+export interface RedisNodeType {
+    readonly name: string;
+    /** Memory, in GiB, as the cloud publishes it for the node type. */
+    readonly memoryGiB: number;
+    /**
+     * SSD, in GiB, on a data-tiering node (ElastiCache's `r6gd`): keys stay in memory, and the values read least
+     * recently move to the SSD. Counted in full beside the memory, as AWS counts a data-tiering node's capacity.
+     */
+    readonly ssdGiB?: number;
+    /** Price per node-hour, in USD. */
+    readonly hourlyUSD: number;
+    /**
+     * The most shards a cluster of this type is priced at. The default catalogue prices its burstable nodes as one
+     * shard: a cluster sharded across them is not how data anyone queries is held. Default: no limit.
+     */
+    readonly maxShards?: number;
+}
+/**
+ * Redis sized to hold the data: enough shards for the bytes, each a primary and its replicas, on whichever node
+ * type in `nodeTypes` makes that cheapest. Prices within a millionth of a dollar of the cheapest are a tie, which
+ * fewer nodes win, then the lower price, then the name, so the answer never depends on the order of the rows. The data is held at its stored,
+ * compressed size, which is a floor on the memory Redis needs for it: a native Redis bitmap is sized by its highest
+ * id, not by how many ids it holds, so sparse ids need more.
+ */
+export interface RedisSizing {
+    /** Where the prices come from, and when; the report's notes quote it. */
+    readonly source: string;
+    readonly nodeTypes: readonly RedisNodeType[];
+    /** Replicas per shard. AWS's best practice is 2; Multi-AZ needs at least 1. */
+    readonly replicasPerShard: number;
+    /** The share of each node's memory kept back from data: ElastiCache's `reserved-memory-percent`, 25% by default. */
+    readonly reservedMemoryFraction: number;
+}
+/**
+ * **ElastiCache for Redis OSS in us-east-1, on-demand**: node prices from AWS's public price list, version
+ * 20260914063714 (published 2026-09-14, effective 2026-09-01,
+ * https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonElastiCache/20260914063714/us-east-1/index.json), and the
+ * memory AWS publishes for each node type. Every shard is a primary and two replicas, AWS's best practice, with
+ * ElastiCache's default 25% of each node's memory reserved, and the burstable `t4g` nodes are priced as one shard.
+ *
+ * It lists only rows that are the cheapest fit for some size of data: the data-tiering `r6gd` rows above about
+ * 30 GiB, and the larger in-memory `r6g` rows, `2xlarge` and up, when those are left out. The current `m7g` and `r7g` nodes are not here,
+ * because an `m6g` or `r6g` with the same memory costs less. It is the cheapest cluster of THIS kind; Redis bought
+ * another way costs less — one replica a shard a third less, ElastiCache for Valkey 20% less a node, reserved nodes
+ * less again — so against those the saving the verdict reports is smaller. Pass their prices to compare with them.
+ */
+export declare const ELASTICACHE_REDIS_US_EAST_1_ONDEMAND: RedisSizing;
+/**
+ * One ElastiCache HA cluster, a primary and two replicas of cache.m7g.large — the m7g family's smallest node —
+ * at $0.158 an hour each: 3 × 730 h × $0.158 = $346.02, published to the dollar. The per-request crossovers on the
+ * benchmarks page are drawn against this one cluster, whatever the data size; pass it as `pricing.redis` to compare
+ * with it. It is not the cheapest cluster for any size of data: three cache.m6g.large hold the same memory for less.
+ */
+export declare const ONE_REDIS_HA_CLUSTER: Readonly<{
+    monthlyUSD: number;
+}>;
+/**
+ * Default profile — **AWS us-east-1, on-demand**, mid-2026, from the fact-checked published pricing rather than
+ * copied from a blog post, with Redis sized to the data. Override it for your region, cloud, or committed term.
+ */
+export declare const AWS_US_EAST_1_ONDEMAND: PricingProfile;
+/** Sustained access pattern. All rates default to 0; unspecified ⇒ that op contributes nothing. */
+export interface Workload {
+    /**
+     * Point reads (`has`) per second. Each cache miss is one object GET, once the segment's pointer and index are
+     * read.
+     */
+    readonly readsPerSec?: number;
+    readonly intersectsPerSec?: number;
+    /** Cache hit rate in `[0, 1]` — hits are free; only misses cost. Default 0. */
+    readonly cacheHitRate?: number;
+    /**
+     * Storage chunks one intersection fetches, summed over its operands: the chunk-skipping survivors. Default 1.
+     * Two segments sharing `k` chunks fetch `2k`. The model adds each operand's pointer and index reads itself (see
+     * {@link Workload.operandsPerIntersect}), so count chunks only. One more GET per operand whose index outgrows
+     * the reader's tail read (256 KiB by default), which then reads the index whole, belongs here too, and so does a
+     * pointer re-read by an intersect slow enough to outlive {@link Workload.genTtlMs}.
+     */
+    readonly chunksPerIntersect?: number;
+    /**
+     * Segments each intersection reads, `exclude` operands included; at least 1. Default 2. An intersection is priced
+     * **cold**: before its chunks, each operand's pointer is read, then its index, in one read of the object's tail —
+     * 2 GETs an operand, so a cold intersect of two segments sharing `k` chunks is `4 + 2k` GETs. `cacheHitRate`
+     * does not apply to intersections, so a long-lived reader that answers a repeat from its cache pays less. Other
+     * combines read their operands the same way and can be priced here too, with the chunks they fetch.
+     */
+    readonly operandsPerIntersect?: number;
+    /**
+     * Generations published per month across the modeled data — the write side of a loaded store. Default **0**
+     * ⇒ loads are not modeled and the report *discloses* the omission rather than silently under-reporting.
+     */
+    readonly loadsPerMonth?: number;
+    /**
+     * PUT-class requests one load's object write issues, each priced at the PUT rate. Default **1** (a single-object
+     * PUT). A multipart write of `P` parts bills `P + 2` (initiate, the parts, complete) — set it when you know your
+     * object sizes. The model adds what `store.load()` does around the write: two listings and the pointer's write,
+     * PUT-class on S3, and nine GETs, the pointer read eight times and the current generation's index once. That is
+     * a segment with two generations behind it; its first load makes two fewer GETs, and its second one fewer. On
+     * S3 at the default prices a single-part `store.load()` is then about $23.60 per million. A segment whose index
+     * outgrows the tail read makes one more GET, and a publish that loses a race to another writer reads the
+     * pointer again.
+     */
+    readonly requestsPerLoad?: number;
+    /**
+     * Segments each long-lived reader process keeps reading. A reader re-reads a segment's pointer when it reads the
+     * segment after {@link Workload.genTtlMs} has passed, so each costs at most one GET per `genTtlMs` in each process
+     * that reads it — 1,314,000 GETs a month at the default 2 s — and the whole term at most one GET per point read
+     * (`readsPerSec`, across the fleet). Intersections are priced cold and pay their own pointer reads. Default **0**
+     * ⇒ the refresh is not modeled, and the report *discloses* the omission when there are reads to refresh for.
+     *
+     * It assumes each hot segment stays open in the reader's cache (`cache.readerMax`, 1,024 segments by default,
+     * and `cache.readerMaxBytes`, 64 MiB of parsed index). A read of a segment the cache evicted opens it again, a
+     * pointer read and a tail read, which is not priced here, and neither is the re-open every reader makes after
+     * each load, for the new generation's index. Size those caches to keep the hot set open; the report says so when
+     * `hotSegments` is more than a store keeps open by default.
+     */
+    readonly hotSegments?: number;
+    /**
+     * Long-lived reader processes, each keeping its own {@link Workload.hotSegments} open and refreshing their
+     * pointers on its own: ten processes reading the same hundred segments pay ten times one process's refresh.
+     * At least 1. Default 1.
+     */
+    readonly readerProcesses?: number;
+    /**
+     * How long the reader trusts a pointer, in ms: the store's `cache.genTtlMs`. Default 2000, the store's own
+     * default; `segment.costReport()` uses the store's. `0` turns the timed refresh off, and the model bills none: a
+     * reader then re-reads a pointer only when its cache evicts the segment, a read finds its generation swept, or a
+     * write invalidates it, none of which this term prices.
+     */
+    readonly genTtlMs?: number;
+}
+/** One (group of) segment(s) for planning. `count` = how many like this (default 1). */
+export interface SegmentSizing {
+    readonly sizeBytes?: number;
+    readonly cardinality?: number;
+    /** Number of segments with these characteristics. Default 1. */
+    readonly count?: number;
+}
+export interface EstimateInput {
+    readonly segments: readonly SegmentSizing[];
+    readonly workload?: Workload;
+    readonly pricing?: PricingProfile;
+}
+export interface CostReport {
+    readonly monthlyUSD: {
+        readonly byOp: {
+            readonly reads: number;
+            readonly intersects: number;
+            readonly storage: number;
+            /**
+             * Loads: each its object's `requestsPerLoad` PUT-class requests plus what `store.load()` adds around them.
+             * 0 unless `loadsPerMonth` is set.
+             */
+            readonly loads: number;
+            /** The pointer refresh of {@link Workload.hotSegments}. 0 unless `hotSegments` is set. */
+            readonly pointerRefresh: number;
+        };
+        readonly total: number;
+    };
+    /**
+     * The Redis the verdict compares against. Sized, the default, it is the cheapest cluster in
+     * `pricing.redis.sizedToData` that holds this report's stored bytes; fixed, it is `pricing.redis.monthlyUSD` as
+     * given.
+     *
+     * **It is not additive across reports.** Each report sizes Redis to its own bytes, so a per-segment report
+     * (`segment.costReport()`) compares with a cluster holding that one segment alone, and the baselines of a store's
+     * segments do not sum to the store's. To judge a store, price all its segments in one `estimateCost`. To alarm on
+     * one, sum `monthlyUSD.total` over its segments and compare the sum with the Redis you would run for it: a
+     * per-segment verdict against that whole price fires only when one segment alone costs more than all of it.
+     */
+    readonly redisBaseline: {
+        readonly basis: 'fixed';
+        readonly monthlyUSD: number;
+    } | {
+        readonly basis: 'sized-to-data';
+        readonly monthlyUSD: number;
+        /** The node type, the shards, the nodes (every shard's primary and replicas), and whether they tier to SSD. */
+        readonly cluster: {
+            readonly nodeType: string;
+            readonly shards: number;
+            readonly nodes: number;
+            readonly dataTiering: boolean;
+        };
+    };
+    /**
+     * Sustained read rate at which the pay-per-use model's cost passes {@link CostReport.redisBaseline}, **evaluated
+     * at this report's `cacheHitRate`** — so a higher cache-hit rate raises it (cache hits are free) — with the other
+     * request axes at 0, and with what keeping the data readable costs taken out of the baseline first: storage, and
+     * this report's pointer refresh. Loads, the write side, are left out of it. `Infinity` means it never crosses (a
+     * 100% cache-hit rate). The published anchor (~329 reads/s) is against {@link ONE_REDIS_HA_CLUSTER}, at
+     * `cacheHitRate: 0` with no refresh modeled.
+     */
+    readonly redisCrossover: {
+        readonly readsPerSec: number;
+    };
+    readonly verdict: 'win-big' | 'win' | 'lose-zone';
+    readonly rationale: string;
+    readonly assumptions: {
+        readonly cacheHitRate: number;
+        readonly pricingName: string;
+        /** True when segment **sizes** were real (grounded `costReport`), false for a pure `estimateCost`. */
+        readonly grounded: boolean;
+        /** What the model assumed and what it left out, in words. The last says how the Redis was priced. */
+        readonly notes: readonly string[];
+    };
+}
+/**
+ * **Planning** cost estimate — pure, no instance or live data needed (sizing, sales, what-if). Segment sizes
+ * are taken as given (or roughly derived from cardinality); use the grounded `segment.costReport()` for
+ * exact, real sizes. See {@link CostReport}.
+ */
+export declare function estimateCost(input: EstimateInput): CostReport;
+/**
+ * **Grounded** report from a real segment byte total (from the `.crbm` index) + a supplied workload. Used by
+ * `Segment.costReport()` in the facade. `grounded` defaults to true (the size is exact, not estimated); the
+ * caller passes `grounded: false` + a note when the Storage source can't measure size.
+ */
+export declare function groundedReport(input: {
+    readonly storageBytes: number;
+    readonly grounded?: boolean;
+    readonly workload?: Workload;
+    readonly pricing?: PricingProfile;
+    readonly extraNotes?: readonly string[];
+}): CostReport;

package/dist/core/crbm/crc32c.d.ts

@@ -0,0 +1,10 @@
+/**
+ * CRC32C (Castagnoli, polynomial 0x1EDC6F41) — the pre-deserialize integrity screen for `.crbm`
+ * payloads, index, and footer.
+ *
+ * Pure TypeScript, table-driven: `core/` may not import `crypto`/`node:crypto` (the determinism lint
+ * enforces it), and a from-scratch implementation is what the cross-language golden corpus will be pinned
+ * against. CRC32C (not CRC32) is the locked choice — it matches the hardware-accelerated checksum used
+ * by SSE/iSCSI/Btrfs, and it is the algorithm the `.crbm` format locks in.
+ */
+export declare function crc32c(bytes: Uint8Array, seed?: number): number;

package/dist/core/crbm/format.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Frozen `.crbm` v1.0 layout constants.
+ *
+ * These byte widths/offsets are pinned by the golden corpus and must never change for v1 —
+ * a new layout is a new format version. All multi-byte integers are little-endian (v1 fixes LE).
+ */
+/** 4-byte magic at both file ends: "CRBM". */
+export declare const MAGIC: Uint8Array<ArrayBuffer>;
+export declare const VERSION_MAJOR = 1;
+export declare const VERSION_MINOR = 0;
+/** Front preamble: magic(4) + version_major(1) + version_minor(1) + reserved(2). */
+export declare const PREAMBLE_BYTES = 8;
+/** Payloads begin immediately after the preamble. */
+export declare const PAYLOAD_START = 8;
+/** Fixed footer size (v1.0). */
+export declare const FOOTER_BYTES = 104;
+/** Byte offsets of each field within the 104-byte footer. */
+export declare const FOOTER: {
+    readonly indexOffset: 0;
+    readonly indexLength: 8;
+    readonly indexCrc32c: 16;
+    readonly flags: 20;
+    readonly payloadCodecId: 24;
+    readonly elementWidth: 26;
+    readonly containerCodec: 27;
+    readonly versionMajor: 28;
+    readonly versionMinor: 29;
+    readonly reserved2: 30;
+    readonly generation: 32;
+    readonly indexNonce: 40;
+    readonly indexTag: 52;
+    readonly keyId: 68;
+    readonly chunkCount: 84;
+    readonly totalCardinality: 88;
+    readonly footerCrc32c: 96;
+    readonly endMagic: 100;
+};
+/** Footer bytes covered by `footer_crc32c` (everything before the CRC field). */
+export declare const FOOTER_CRC_COVERAGE: 96;
+/** `flags` bit positions. */
+export declare const FLAG_ENCRYPTED: number;
+export declare const FLAG_INDEX_COMPRESSED: number;
+export declare const FLAG_LITTLE_ENDIAN: number;
+/** Flag bits a v1 reader understands; any bit outside this mask is an unsupported feature. */
+export declare const KNOWN_FLAGS: number;
+/**
+ * Default element width: 32-bit ids (the u32 member space). `64` is the *reserved* escape above the
+ * ~4.29 B-ids/segment u32 ceiling; a v1 (32-bit) reader must **reject** a 64-bit generation rather than feed
+ * it to the 32-bit deserializer (which would mis-count fleet-wide). A future 64-bit generation is therefore a
+ * **major**-version bump (auto-rejected by old readers), never a silent minor one — see
+ * {@link CrbmReader.open} validates this.
+ */
+export declare const ELEMENT_WIDTH_32 = 32;
+/**
+ * **Which codec produced the chunk payloads.** Recorded in the footer, validated on read.
+ *
+ * `.crbm` is a shared container: the index, the CRC32Cs, the AEAD framing and the generation model are all
+ * codec-independent, and **only the chunk payload bytes belong to a flavor** (hence *Chunked Remote BitMap* —
+ * see the `.crbm` format section of the API reference). This field is what lets one container hold either, the same way ZIP tags each member
+ * with a compression method.
+ *
+ * **A codec id, not a roaring-specific one.** A second codec is genuinely expected — `soaring` is a planned
+ * Roaring *variant*, so its serialized bytes are unlikely to be roaring-portable, and it lands *after* the `1.0`
+ * format freeze. A field frozen under a codec-specific meaning could not be reinterpreted later without a major
+ * format version.
+ *
+ * **Ids are permanent once published.** Add to {@link KNOWN_PAYLOAD_CODEC_IDS} when a codec ships; never
+ * reuse or renumber. Ids are deliberately *not* pre-allocated for codecs that do not exist — a reserved number
+ * for an unbuilt codec is a guess about a format nobody has designed.
+ */
+export declare const PAYLOAD_CODEC_ROARING_PORTABLE = 1;
+/**
+ * Every payload codec id this reader can decode.
+ *
+ * The reader validates membership rather than equality with a single constant. That is the whole point of an
+ * id registry: an unknown id is rejected with a typed error naming it, so an old reader meeting a
+ * future-codec generation **fails closed** — the correct direction, and the reason a store built on one codec
+ * can never silently misread another's bytes as its own. (The homogeneity contract means one store is one
+ * codec, so meeting a foreign generation implies misconfiguration, and a loud rejection is exactly what you
+ * want there.)
+ */
+export declare const KNOWN_PAYLOAD_CODEC_IDS: ReadonlySet<number>;
+/**
+ * The only container codec v1 defines: `0` = none (payloads are stored as the portable roaring serialization
+ * with no extra container-level transform). A non-zero codec is a future format feature; a v1 reader rejects
+ * it ({@link UnsupportedError}) rather than mis-decode. Stamped by the writer, validated by the reader.
+ */
+export declare const CONTAINER_CODEC_NONE = 0;
+/**
+ * Default speculative-tail-read size. The spec's worst-case index (a full 16-bit span) is ~400–650 KB,
+ * so 256 KB collapses the read to a single GET for the vast majority of real (sparse/medium) segments;
+ * only near-full-span segments take the documented second GET.
+ */
+export declare const DEFAULT_TAIL_BYTES: number;
+/**
+ * Hard cap on the index region a reader will fetch/parse from one (untrusted) object. The index is read
+ * whole on the cheap `open()`/`count()` path, so it must be bounded before allocation — generously
+ * above the ~650 KB full-span worst case, well below a denial-of-wallet allocation.
+ */
+export declare const DEFAULT_MAX_INDEX_BYTES: number;
+/** Hard cap on a single chunk payload, defending the native deserializer against oversized input. */
+export declare const DEFAULT_MAX_PAYLOAD_BYTES: number;
+/** A chunk's cardinality is in `[1, 65536]` (empty chunks are never written). */
+export declare const MAX_CHUNK_CARDINALITY = 65536;
+/** Fixed width of a per-chunk CRC32C field in the index (high-entropy → not varint). */
+export declare const CRC32C_BYTES = 4;
+/**
+ * v1 AEAD framing sizes (AES-256-GCM), fixed by the format exactly like the payload codec id — a
+ * different cipher is a new format version. An encrypted chunk payload is stored as `nonce ‖ ciphertext ‖ tag`;
+ * the encrypted **index**'s nonce/tag live in the footer's reserved `indexNonce`/`indexTag` slots. Matches
+ * {@link FOOTER.indexNonce} (12 B) and {@link FOOTER.indexTag} (16 B).
+ */
+export declare const AEAD_NONCE_BYTES = 12;
+export declare const AEAD_TAG_BYTES = 16;

package/dist/core/crbm/reader.d.ts

@@ -0,0 +1,86 @@
+import type { BlobReader } from '../blob.js';
+import type { CrbmCrypto } from '../crypto.js';
+export interface CrbmIndexEntry {
+    readonly chunkKey: number;
+    readonly offset: number;
+    readonly length: number;
+    readonly cardinality: number;
+    readonly crc32c: number;
+}
+export interface CrbmReaderOptions {
+    /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
+    readonly tailBytes?: number;
+    /** Hard cap on a single chunk payload length (default 16 MB). */
+    readonly maxPayloadBytes?: number;
+    /** Hard cap on the whole index region fetched/parsed from one object (default 8 MB). */
+    readonly maxIndexBytes?: number;
+    /** Decryption context for an encrypted object (its DEK's AEAD + AAD builder). Required iff `FLAG_ENCRYPTED`. */
+    readonly crypto?: CrbmCrypto;
+    /**
+     * Opaque marker for the *incarnation of the name* this object belongs to — the caller's own identity for the
+     * registry row it resolved. The reader never interprets it; it carries it so a caller holding an open reader
+     * can tell "the same generation of the same segment" from "the same generation NUMBER of a segment that was
+     * deleted and re-created", which the number alone cannot express.
+     */
+    readonly lineage?: unknown;
+}
+export declare class CrbmReader {
+    private readonly blob;
+    private readonly objectSize;
+    readonly generation: number;
+    /** See {@link CrbmReaderOptions.lineage} — carried, never interpreted. */
+    readonly lineage: unknown;
+    readonly totalCardinality: number;
+    private readonly entries;
+    private readonly orderedKeys;
+    /** True when `open()` satisfied the index from the tail GET alone (no second range read). */
+    readonly servedFromTail: boolean;
+    /** Set iff the object is encrypted — used to decrypt each chunk payload in {@link getChunk}. */
+    private readonly crypto;
+    /** The footer's own CRC, as verified at open: it covers the index's CRC, the chunk count and the generation. */
+    private readonly footerCrc;
+    private constructor();
+    /**
+     * Which object this reader opened, as its size and its footer's CRC. The footer covers the index's CRC, so two
+     * objects with the same fingerprint hold the same chunks, as far as a CRC can tell; a pin compares it to know that
+     * the object it reopens is the one it pinned, since a purged and reloaded name reuses the generation number.
+     */
+    get fingerprint(): string;
+    /** Total object bytes (from the one-GET tail read) — for grounded storage cost. */
+    get sizeBytes(): number;
+    /**
+     * Estimated retained JS heap of this reader's parsed index — the weight the storage reader cache bounds on
+     * (a wide segment's parsed index, not its payloads, dominates the reader's footprint). `entries.size` scales with the number
+     * of resident chunks (≤ 65536), so this is `entries.size × {@link RETAINED_BYTES_PER_INDEX_ENTRY}`.
+     */
+    get retainedIndexBytes(): number;
+    /** Per-chunk cardinality (`chunkKey → count`) from the parsed index — no payload reads. */
+    cardinalities(): Map<number, number>;
+    /**
+     * Whether the object behind `blob` is the one `fingerprint` names ({@link fingerprint}), from one tail read of a
+     * footer's worth, with no key and no index, since the footer is stored in the clear and checked by its own CRC.
+     * Another size is another object, whatever its bytes hold: a short one that is no `.crbm` at all is not the one
+     * pinned. At the same size the footer's CRC says, and a footer that fails its own checks says nothing about which
+     * object is there, so this throws, as an open would. What a pin asks, to tell the object it opened from one that
+     * has since been stored under its key, whoever wrote it.
+     */
+    static sameObject(blob: BlobReader, fingerprint: string): Promise<boolean>;
+    static open(blob: BlobReader, options?: CrbmReaderOptions): Promise<CrbmReader>;
+    /** Chunk keys present in this generation, ascending. */
+    chunkKeys(): number[];
+    /** Segment cardinality from the footer (no payload reads) — the cheap `count()` path. */
+    count(): number;
+    has(chunkKey: number): boolean;
+    /**
+     * Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
+     * absent. Throws `IntegrityError` on a CRC mismatch (the bytes must never reach the native
+     * deserializer). The returned buffer is a **read-only view** owned by the reader/driver — callers must
+     * not mutate it.
+     */
+    getChunk(chunkKey: number): Promise<Uint8Array | null>;
+}
+export declare function parseIndex(indexBytes: Uint8Array, payloadEnd: number, maxPayloadBytes: number, minPayloadBytes?: number): {
+    entries: Map<number, CrbmIndexEntry>;
+    orderedKeys: number[];
+    cardinalitySum: number;
+};

package/dist/core/crbm/varint.d.ts

@@ -0,0 +1,13 @@
+/** Append the LEB128 encoding of a u32 `value` to `out`. */
+export declare function writeVarint(out: number[], value: number): void;
+export interface VarintRead {
+    readonly value: number;
+    /** Offset just past the bytes consumed. */
+    readonly next: number;
+}
+/**
+ * Decode a LEB128 varint from `bytes` starting at `offset`. Throws `IntegrityError` on a truncated or
+ * over-long (> 5-byte / > u32) encoding — the bytes are untrusted, so a malformed index must fail
+ * cleanly, never loop or overflow.
+ */
+export declare function readVarint(bytes: Uint8Array, offset: number): VarintRead;