@cloudbitmaps/core 0.11.2 → 0.13.0

package/dist/core/cost.d.ts

@@ -8,9 +8,19 @@ export interface PricingProfile {
         readonly putPerMillion: number;
         readonly storagePerGiBMonth: number;
         /**
-         * Requests one read costs when it needs the object's size first: a pointer read, and a segment's tail read.
-         * Default **1**, S3's, whose suffix-range GET returns the size with the bytes. **2** on Azure Blob,
-         * which reads the properties and then the bytes; GCS answers in one request, like S3. A chunk read knows its range, and is one request everywhere.
+         * Requests one pointer read costs: a read of a segment's registry row, whose version comes back with its bytes.
+         * Default **1**, and 1 on S3, GCS and Azure Blob, each of which answers it with one GET. Charged for each operand
+         * of an intersection, for the pointer reads a load makes, and for each pointer refresh. Set it when a
+         * registry of your own takes more than one request to read a row.
+         */
+        readonly requestsPerPointerRead?: number;
+        /**
+         * Requests one tail read costs: the read of a segment's index from the end of its generation, which needs the
+         * object's size. Default **1**, S3's, whose suffix-range GET returns the size with the bytes, and GCS's, the same.
+         * **2** on Azure Blob, which takes no suffix range and reads the properties and then the bytes. Charged for each
+         * operand of an intersection. A load reads no index of its own: a row that carries a summary of its current
+         * generation gives the load the size it needs, and the checks it makes are one metadata request on every backend.
+         * A chunk read knows its range, and is one request everywhere.
          */
         readonly requestsPerSizedRead?: number;
     };
@@ -110,7 +120,8 @@ export interface Workload {
     /**
      * Segments each intersection reads, `exclude` operands included; at least 1. Default 2. An intersection is priced
      * **cold**: before its chunks, each operand's pointer is read, then its index, in one read of the object's tail —
-     * 2 GETs an operand, so a cold intersect of two segments sharing `k` chunks is `4 + 2k` GETs. `cacheHitRate`
+     * 2 GETs an operand on S3 and GCS, so a cold intersect of two segments sharing `k` chunks is `4 + 2k` GETs there,
+     * and 3 on Azure Blob, whose tail read is two requests (see {@link PricingProfile}). `cacheHitRate`
      * does not apply to intersections, so a long-lived reader that answers a repeat from its cache pays less. Other
      * combines read their operands the same way and can be priced here too, with the chunks they fetch.
      */
@@ -123,12 +134,23 @@ export interface Workload {
     /**
      * PUT-class requests one load's object write issues, each priced at the PUT rate. Default **1** (a single-object
      * PUT). A multipart write of `P` parts bills `P + 2` (initiate, the parts, complete) — set it when you know your
-     * object sizes. The model adds what `store.load()` does around the write: two listings and the pointer's write,
-     * PUT-class on S3, and nine GETs, the pointer read eight times and the current generation's index once. That is
-     * a segment with two generations behind it; its first load makes two fewer GETs, and its second one fewer. On
-     * S3 at the default prices a single-part `store.load()` is then about $23.60 per million. A segment whose index
-     * outgrows the tail read makes one more GET, and a publish that loses a race to another writer reads the
-     * pointer again.
+     * object sizes. The model adds what `store.load()` does around the write, at the default `keep` of 1: the pointer's
+     * write, PUT-class on S3, and five GETs: the pointer read three times, one check that the next generation number is
+     * free, and one that the current generation's object is there, which tells the collection that deletes by name it
+     * may. The current generation's size comes from the row's summary of it, so the load reads no index. Collection
+     * deletes the generation the window pushed out by name, so it lists only on every 16th generation, which adds a
+     * PUT-class request and two pointer reads there and makes no check that the current generation is there, a
+     * sixteenth of each on average. That is a segment with two
+     * generations behind it, whose row carries a summary; its first two loads make fewer requests and collect nothing,
+     * and the first load of a row written before rows carried a summary reads the current generation's index, one tail
+     * read, in place of the check that its object is there. A publish that loses a race to another writer
+     * reads the pointer again, and a load whose check finds the number taken (a crashed load's object, or the
+     * generations a rollback left above the pointer) lists the segment's objects to number past them and to collect,
+     * two PUT-class requests on S3 and two more pointer reads. So does every load that keeps two or more generations,
+     * which lists to collect: one more PUT-class request and two more pointer reads than the model counts. The counts
+     * are a cleartext segment's: an encrypted segment's load reads its row once more, after its ids and before it
+     * unwraps the key, one more GET ($0.40 per million at the default prices) that the model leaves out, beside the
+     * key-management calls it does not price either.
      */
     readonly requestsPerLoad?: number;
     /**
@@ -158,6 +180,26 @@ export interface Workload {
      * write invalidates it, none of which this term prices.
      */
     readonly genTtlMs?: number;
+    /**
+     * Segments the retention sweep (`retireExpired`) retires a month. Each costs 9 registry reads, 3 writes and a
+     * delete with {@link Workload.conditionalDelete} on, and 8 reads and 3 writes with it off, priced at the GET and PUT
+     * rates, a delete unbilled, as S3 leaves it. Default
+     * **0**.
+     */
+    readonly retirementsPerMonth?: number;
+    /**
+     * Tombstones the sweep purges a month: at a steady state, as many as it retires a month, a `tombstoneGraceMs` later.
+     * A purge costs 4 reads and 2 deletes with {@link Workload.conditionalDelete} on, and 3 reads and a write with it
+     * off. Default **0**.
+     */
+    readonly purgesPerMonth?: number;
+    /**
+     * Whether the registry's deletes remove a row for good (`RegCaps.conditionalDelete`), which the shipped registries
+     * do where the backend applies a delete precondition. Default **true**. `false` prices the sweep of a registry that
+     * only tombstones: a purge rewrites the row instead of deleting it, and every later full sweep reads each tombstone
+     * left (two reads per purged segment), which this term does not price.
+     */
+    readonly conditionalDelete?: boolean;
 }
 /** One (group of) segment(s) for planning. `count` = how many like this (default 1). */
 export interface SegmentSizing {
@@ -184,6 +226,11 @@ export interface CostReport {
             readonly loads: number;
             /** The pointer refresh of {@link Workload.hotSegments}. 0 unless `hotSegments` is set. */
             readonly pointerRefresh: number;
+            /**
+             * The retention sweep's retirements and purges: each's registry reads at the GET rate and its writes at the PUT
+             * rate. 0 unless `retirementsPerMonth` or `purgesPerMonth` is set.
+             */
+            readonly retention: number;
         };
         readonly total: number;
     };
@@ -234,6 +281,40 @@ export interface CostReport {
         readonly notes: readonly string[];
     };
 }
+/**
+ * What the retention sweep makes per segment, counted with a store that counts its requests
+ * (`tests/core/retention-hard-purge.test.ts` holds each figure to the engine, and `tests/core/cost.test.ts` holds the
+ * estimator to these). With the registry's `conditionalDelete` on, a retirement files a due-index pointer to the
+ * tombstone and a purge removes the row and every pointer: a later sweep reads nothing of a purged segment. With it
+ * off, a retirement files no pointer, a purge rewrites the row as a tombstone, and every later full sweep reads what
+ * is left, two objects per purged segment.
+ */
+export declare const RETENTION_SWEEP_REQUESTS: Readonly<{
+    conditionalDelete: {
+        retirement: {
+            reads: number;
+            writes: number;
+            deletes: number;
+        };
+        purge: {
+            reads: number;
+            writes: number;
+            deletes: number;
+        };
+    };
+    tombstoning: {
+        retirement: {
+            reads: number;
+            writes: number;
+            deletes: number;
+        };
+        purge: {
+            reads: number;
+            writes: number;
+            deletes: number;
+        };
+    };
+}>;
 /**
  * **Planning** cost estimate — pure, no instance or live data needed (sizing, sales, what-if). Segment sizes
  * are taken as given (or roughly derived from cardinality); use the grounded `segment.costReport()` for

package/dist/core/crbm/format.d.ts

@@ -1,5 +1,5 @@
 /**
- * Frozen `.crbm` v1.0 layout constants.
+ * Frozen `.crbm` v1 layout constants (format 1.0, with its optional extension block).
  *
  * These byte widths/offsets are pinned by the golden corpus and must never change for v1 —
  * a new layout is a new format version. All multi-byte integers are little-endian (v1 fixes LE).
@@ -41,6 +41,12 @@ export declare const FOOTER_CRC_COVERAGE: 96;
 export declare const FLAG_ENCRYPTED: number;
 export declare const FLAG_INDEX_COMPRESSED: number;
 export declare const FLAG_LITTLE_ENDIAN: number;
+/**
+ * The object carries an extension block just before its index (a generation's metadata). Set only when there is
+ * something to put in it, so an object without metadata is the same bytes as before the block existed; a reader that
+ * does not know the bit refuses the object rather than reading past what it cannot see.
+ */
+export declare const FLAG_EXTENSION: number;
 /** Flag bits a v1 reader understands; any bit outside this mask is an unsupported feature. */
 export declare const KNOWN_FLAGS: number;
 /**
@@ -98,10 +104,37 @@ export declare const DEFAULT_TAIL_BYTES: number;
  * above the ~650 KB full-span worst case, well below a denial-of-wallet allocation.
  */
 export declare const DEFAULT_MAX_INDEX_BYTES: number;
-/** Hard cap on a single chunk payload, defending the native deserializer against oversized input. */
-export declare const DEFAULT_MAX_PAYLOAD_BYTES: number;
+/**
+ * Largest serialized bitmap the engine decodes (1 MiB, per chunk). A well-formed 16-bit chunk serializes to at most
+ * about 8.2 KiB, so no legitimate chunk is near it; both the read engine and the erase path decode under this one cap.
+ */
+export declare const DEFAULT_MAX_BITMAP_BYTES: number;
 /** A chunk's cardinality is in `[1, 65536]` (empty chunks are never written). */
 export declare const MAX_CHUNK_CARDINALITY = 65536;
+/**
+ * The extension block, present when the footer sets {@link FLAG_EXTENSION}: `section* ‖ u32 sectionsLength ‖ u32
+ * crc32c ‖ "CRBX"`, between the last payload and the index, so the index starts right after it. A reader finds it from
+ * `indexOffset`: the trailer is the {@link EXT_TRAILER_BYTES} bytes before the index, and the CRC32C covers the sections
+ * and the length field. Each section is `u8 type ‖ u32 length ‖ bytes`, in strictly ascending type order; a reader
+ * skips a type it does not know, so a section must be safe to ignore.
+ */
+export declare const EXT_MAGIC: Uint8Array<ArrayBuffer>;
+/** The trailer: the sections' length (u32), their CRC32C (u32) and {@link EXT_MAGIC}. */
+export declare const EXT_TRAILER_BYTES = 12;
+/** A section's header: its type (u8) and its length (u32). */
+export declare const EXT_SECTION_HEADER_BYTES = 5;
+/**
+ * Cap on the sections of one block, in bytes. A reader refuses a larger block from its trailer alone, before it reads
+ * any sections beyond what it already holds (the read that brings the index may already hold up to this much before
+ * it), and a writer of any 1.x minor keeps within it, so a later section type stays readable by this reader.
+ */
+export declare const MAX_EXT_BYTES: number;
+/**
+ * Section type 1: the generation's metadata, its canonical JSON in UTF-8. On an encrypted object the section is
+ * `nonce ‖ ciphertext ‖ tag`, sealed under the segment's key with the metadata scope of the associated data.
+ * Type 0 is not a section type: a reader refuses it.
+ */
+export declare const EXT_SECTION_METADATA = 1;
 /** Fixed width of a per-chunk CRC32C field in the index (high-entropy → not varint). */
 export declare const CRC32C_BYTES = 4;
 /**
@@ -112,3 +145,9 @@ export declare const CRC32C_BYTES = 4;
  */
 export declare const AEAD_NONCE_BYTES = 12;
 export declare const AEAD_TAG_BYTES = 16;
+/**
+ * Hard cap on one encrypted chunk's stored payload (a cleartext one is capped at the decode cap): the decode cap plus the AEAD framing, so an
+ * entry the engine would refuse to decode is refused when the object is opened, before any payload is read. No writer
+ * exceeds it: a stored payload is a serialized 16-bit chunk (at most about 8.2 KiB) plus, when encrypted, that framing.
+ */
+export declare const DEFAULT_MAX_PAYLOAD_BYTES: number;

package/dist/core/crbm/reader.d.ts

@@ -1,5 +1,6 @@
 import type { BlobReader } from '../blob.js';
 import type { CrbmCrypto } from '../crypto.js';
+import type { GenerationMetadata } from '../ports.js';
 /**
  * A parsed index as parallel typed arrays, one slot per entry, ascending by key. Typed arrays hold each field at
  * its own width with no per-entry object, so the retained heap is the arrays' byte length and nothing else.
@@ -30,11 +31,15 @@ export declare function indexCapacity(indexLength: number): number;
 export interface CrbmReaderOptions {
     /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
     readonly tailBytes?: number;
-    /** Hard cap on a single chunk payload length (default 16 MB). */
+    /** Hard cap on a single chunk payload length (default: the 1 MiB decode cap, plus the 28-byte nonce and tag on an encrypted object). The engine decodes under `maxBitmapBytes`; raise this with it. */
     readonly maxPayloadBytes?: number;
     /** Hard cap on the whole index region fetched/parsed from one object (default 8 MB). */
     readonly maxIndexBytes?: number;
-    /** Decryption context for an encrypted object (its DEK's AEAD + AAD builder). Required iff `FLAG_ENCRYPTED`. */
+    /**
+     * Decryption context for an encrypted object (its DEK's AEAD + AAD builder). Required iff the object's footer sets
+     * `FLAG_ENCRYPTED`: an encrypted object opened without one throws {@link ValidationError}, and an object that is
+     * not encrypted, opened with one, is refused with {@link IntegrityError}.
+     */
     readonly crypto?: CrbmCrypto;
     /**
      * Opaque marker for the *incarnation of the name* this object belongs to — the caller's own identity for the
@@ -44,6 +49,15 @@ export interface CrbmReaderOptions {
      */
     readonly lineage?: unknown;
 }
+/** An object's fingerprint, from its size and its footer's CRC (see {@link CrbmReader.fingerprint}). */
+export declare const fingerprintFor: (size: number, footerCrc: number) => string;
+/**
+ * Whether the object behind `blob` says it is encrypted, from one tail read of a footer's worth, with no key and no
+ * index; a footer that fails its own checks throws, as an open would. For a caller that must not open an object with
+ * the key it holds: an erasure looking for an id in a cleartext object under an encrypted segment, and a rollback
+ * checking that its target is what the row says.
+ */
+export declare function footerSaysEncrypted(blob: BlobReader): Promise<boolean>;
 export declare class CrbmReader {
     private readonly blob;
     private readonly objectSize;
@@ -52,12 +66,22 @@ export declare class CrbmReader {
     readonly lineage: unknown;
     readonly totalCardinality: number;
     private readonly index;
-    /** True when `open()` satisfied the index from the tail GET alone (no second range read). */
+    /**
+     * True when `open()` satisfied the index, and any extension block, from the tail GET alone (no second range
+     * read).
+     */
     readonly servedFromTail: boolean;
     /** Set iff the object is encrypted — used to decrypt each chunk payload in {@link getChunk}. */
     private readonly crypto;
     /** The footer's own CRC, as verified at open: it covers the index's CRC, the chunk count and the generation. */
     private readonly footerCrc;
+    /**
+     * The metadata the generation was written with, checked and decrypted at open; `undefined` when it has none
+     * (every format 1.0 object). Frozen: every caller shares one copy.
+     */
+    readonly metadata: GenerationMetadata | undefined;
+    /** What that metadata adds to {@link retainedBytes}; 0 when there is none. */
+    private readonly metadataWeight;
     private constructor();
     /**
      * Which object this reader opened, as its size and its footer's CRC. The footer covers the index's CRC, so two
@@ -73,6 +97,13 @@ export declare class CrbmReader {
      * arrays that hold it, {@link RETAINED_BYTES_PER_INDEX_ENTRY} per entry.
      */
     get retainedIndexBytes(): number;
+    /**
+     * What this reader holds, the weight the storage reader cache bounds on: its parsed index
+     * ({@link retainedIndexBytes}), plus a weight for the metadata it decoded, from its canonical length and its number
+     * of keys. Equal to {@link retainedIndexBytes} for a generation with no metadata. A cache heuristic, not a contract:
+     * how the metadata is weighed may change in any release.
+     */
+    get retainedBytes(): number;
     /** Per-chunk cardinality (`chunkKey → count`) from the parsed index — no payload reads. */
     cardinalities(): Map<number, number>;
     /**
@@ -100,4 +131,15 @@ export declare class CrbmReader {
      */
     getChunk(chunkKey: number): Promise<Uint8Array | null>;
 }
+/**
+ * Read the sections of an extension block, its bytes before the trailer, once their CRC has passed. Each section is
+ * `u8 type ‖ u32 length ‖ bytes`, in strictly ascending type order, and together they fill the region exactly. Type 1
+ * is the generation's metadata, held to the metadata rules and its canonical form (and opened first with `crypto`,
+ * under the metadata scope, when the object is encrypted); a type this build does not know is a later build's, and
+ * is skipped. Returns the metadata, or `undefined` when the block has none. Every other shape is an
+ * {@link IntegrityError}.
+ *
+ * Exported for the coverage-guided fuzz harness, which drives it directly, past the block's CRC; not public API.
+ */
+export declare function parseExtension(sections: Uint8Array, crypto: CrbmCrypto | undefined): GenerationMetadata | undefined;
 export declare function parseIndex(indexBytes: Uint8Array, payloadEnd: number, maxPayloadBytes: number, minPayloadBytes?: number): ParsedIndex;

package/dist/core/crbm/writer.d.ts

@@ -1,4 +1,5 @@
 import type { CrbmCrypto } from '../crypto.js';
+import type { GenerationMetadata } from '../ports.js';
 import type { BlobSink } from '../blob.js';
 export interface CrbmWriterOptions {
     /** Self-describing generation number (also encoded in the object key). */
@@ -13,6 +14,13 @@ export interface CrbmWriterOptions {
     readonly elementWidth?: number;
     /** When set, payloads + index are AES-256-GCM-encrypted, bound to `(segment, generation, scope)` via AAD. */
     readonly crypto?: CrbmCrypto;
+    /**
+     * The generation's metadata, checked against the metadata rules and copied as its canonical JSON when the writer
+     * is constructed, so a later change to the caller's object never reaches the object. A value that breaks a rule
+     * throws {@link ValidationError} from the constructor, before a byte is written. None, or the empty object, writes
+     * format 1.0.
+     */
+    readonly metadata?: GenerationMetadata;
 }
 export declare class CrbmWriter {
     private readonly sink;
@@ -23,17 +31,38 @@ export declare class CrbmWriter {
     private totalCardinality;
     private preambleWritten;
     private finished;
+    /** The metadata's canonical JSON, copied at construction; `undefined` leaves the footer's extension flag clear. */
+    private readonly metadata;
+    /** Bytes written so far, the footer included once {@link finish} has run. */
+    private written;
+    private footerCrc32c;
     constructor(sink: BlobSink, options: CrbmWriterOptions);
     /**
      * Append one chunk's payload (the roaring-serialized bytes) with its cardinality. `chunkKey` must be
      * strictly greater than the previous one (ascending, no duplicates); `cardinality` is `[1, 65536]`.
      */
     addChunk(chunkKey: number, payload: Uint8Array, cardinality: number): Promise<void>;
-    /** Seal one chunk payload into the on-disk frame `nonce ‖ ciphertext ‖ tag`, bound to its chunkKey. */
-    private sealChunk;
-    /** Write the index region + the fixed footer. After this the object is complete and immutable. */
+    /**
+     * Write the extension block (when there is metadata), the index region and the fixed footer. After this the object
+     * is complete and immutable.
+     */
     finish(): Promise<void>;
+    /**
+     * The finished object's size and its footer's CRC: what names it, as a reader's fingerprint does. A writer that
+     * has to tell its own object from another stored later under the same key compares these with the object's
+     * footer. Throws before {@link finish}.
+     */
+    get identity(): {
+        readonly size: number;
+        readonly footerCrc: number;
+    };
     private ensurePreamble;
+    /**
+     * The extension block: one metadata section (`u8 type ‖ u32 length ‖ bytes`), then the trailer
+     * (`u32 sectionsLength ‖ u32 crc32c(sections ‖ sectionsLength) ‖ "CRBX"`). Encrypted, the section holds
+     * `nonce ‖ ciphertext ‖ tag`, sealed with the metadata scope of the associated data.
+     */
+    private extensionBlock;
     private encodeIndex;
     private buildFooter;
 }