@cloudbitmaps/core 0.5.0 → 0.7.0

package/dist/index.d.cts

@@ -20,9 +20,13 @@ export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s
  * **Interface surface = exactly what the engine needs**, kept general enough for the known fast-follow codecs
  * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitset` will add live on that
  * package's own extended value type, not here — the engine never calls them). The one codec-linked concern
- * this seam intentionally leaves in the format layer is the `.crbm` **serialization id** (`ROARING_PORTABLE_ID`
- * in `crbm/format.ts`): it is stamped in the footer and validated on read, and generalizes to a per-codec id
- * *with* the second codec's format work — a documented follow-up, not part of this interface.
+ * this seam intentionally leaves in the format layer is the `.crbm` **payload codec id**
+ * (`PAYLOAD_CODEC_ROARING_PORTABLE` / `KNOWN_PAYLOAD_CODEC_IDS` in `crbm/format.ts`): it is stamped in the footer
+ * and validated on read. That generalization is **done** as of 0.7.0 — the field was `roaringSerializationId`
+ * and validated by equality against a single constant, which would have frozen a codec-specific name into the
+ * format at `1.0`. It is now a registry the reader checks membership against, so a future codec is a one-line
+ * registration rather than a major format version. Still deliberately outside this interface: a codec declares
+ * its bytes, the seam does not.
  */
 /**
  * The value type a codec produces — a mutable set of `u32` with set algebra and portable (de)serialization.
@@ -61,6 +65,27 @@ interface CodecBitmap {
      * Roaring answers it in O(1) from its container index, so the flagship codec implements it.
      */
     maximum?(): number | undefined;
+    /**
+     * Re-encode for size, in place, immediately before a **cold** write. Representation only — this must never
+     * change membership, and `serialize()` afterwards must decode back to exactly the same set.
+     *
+     * **Optional, like {@link maximum}.** A codec whose encoding has no size decision to make (a plain bitset has
+     * one representation and nothing to choose) simply omits it, and the engine skips the call.
+     *
+     * WHY THIS EXISTS. Roaring picks per container between an array, a bitset and a RUN — but the run choice is
+     * not automatic in any implementation: it is a `runOptimize()` pass you have to ask for, and nothing here was
+     * asking. So two of the three container types were ever used, and run-shaped data paid list or bitmap prices
+     * for a run. Measured on the shipped codec: a contiguous 1,000,000-id range serialized to 128.1 KiB where
+     * run-encoding needs **0.2 KiB** (570×), and a 2,000-run shape 536.5 KiB against **8.5 KiB** (63×). Sparse
+     * data is unchanged, because there are no runs to find — the pass is not a gamble.
+     *
+     * WHY ONLY ON THE COLD PATH. This is called where a whole immutable generation is written, so its cost
+     * amortizes over a write that is already serializing and checksumming every chunk. It is deliberately NOT
+     * called on the warm delta path (`chunk.ts`), which runs per operation: the hot path must not pay for a
+     * rare-ish win, per KISS/YAGNI. Warm rows are short-lived and get folded into a cold generation by
+     * compaction, where they are optimized then.
+     */
+    optimize?(): void;
 }
 /**
  * A pluggable bitmap codec — the factory the codec-agnostic engine constructs {@link CodecBitmap}s through.
@@ -2021,7 +2046,15 @@ interface DestroyResult {
      * `segment.erase` audit event keys off — a cleartext "erase" is not an irreversible destruction.
      */
     readonly cryptoShredded: boolean;
-    /** `'absent'` (no registry row), `'already'` (already destroyed), or `'cleartext'` (no DEK to shred). */
+    /**
+     * Why this segment was not a fresh crypto-shred. `'absent'` (no registry row), `'already'` (already
+     * destroyed), `'cleartext'` (no DEK to shred) — and from `eraseNamespace` only, where per-segment faults are
+     * isolated so one failure cannot discard the ledger: `'contended'` (warm rows were rewritten during every
+     * erase pass) or `` `failed: <message>` `` (any other fault).
+     *
+     * The last two come with `destroyed: false` and mean **the segment still holds data**. A namespace erase
+     * returns them rather than throwing, so its entries have to be inspected — see {@link eraseNamespace}.
+     */
     readonly reason?: string;
     /** Warm rows physically deleted as part of the erase. */
     readonly warmRowsDeleted: number;
@@ -2256,7 +2289,13 @@ declare class RetryingRegistryDriver implements IRegistryDriver {
 interface CrbmWriterOptions {
     /** Self-describing generation number (also encoded in the object key). */
     readonly generation: number;
-    readonly roaringSerializationId?: number;
+    /**
+     * Which codec produced the chunk payloads. Defaults to {@link PAYLOAD_CODEC_ROARING_PORTABLE}.
+     *
+     * Renamed from `roaringSerializationId` in 0.7.0 — same footer field, same offset and width, generalized
+     * because `.crbm` is a shared container and a second codec is expected before the format freezes.
+     */
+    readonly payloadCodecId?: number;
     readonly elementWidth?: number;
     /** When set, payloads + index are AES-256-GCM-encrypted, bound to `(segment, generation, scope)` via AAD. */
     readonly crypto?: CrbmCrypto;

package/dist/index.d.ts

@@ -20,9 +20,13 @@ export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s
  * **Interface surface = exactly what the engine needs**, kept general enough for the known fast-follow codecs
  * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitset` will add live on that
  * package's own extended value type, not here — the engine never calls them). The one codec-linked concern
- * this seam intentionally leaves in the format layer is the `.crbm` **serialization id** (`ROARING_PORTABLE_ID`
- * in `crbm/format.ts`): it is stamped in the footer and validated on read, and generalizes to a per-codec id
- * *with* the second codec's format work — a documented follow-up, not part of this interface.
+ * this seam intentionally leaves in the format layer is the `.crbm` **payload codec id**
+ * (`PAYLOAD_CODEC_ROARING_PORTABLE` / `KNOWN_PAYLOAD_CODEC_IDS` in `crbm/format.ts`): it is stamped in the footer
+ * and validated on read. That generalization is **done** as of 0.7.0 — the field was `roaringSerializationId`
+ * and validated by equality against a single constant, which would have frozen a codec-specific name into the
+ * format at `1.0`. It is now a registry the reader checks membership against, so a future codec is a one-line
+ * registration rather than a major format version. Still deliberately outside this interface: a codec declares
+ * its bytes, the seam does not.
  */
 /**
  * The value type a codec produces — a mutable set of `u32` with set algebra and portable (de)serialization.
@@ -61,6 +65,27 @@ interface CodecBitmap {
      * Roaring answers it in O(1) from its container index, so the flagship codec implements it.
      */
     maximum?(): number | undefined;
+    /**
+     * Re-encode for size, in place, immediately before a **cold** write. Representation only — this must never
+     * change membership, and `serialize()` afterwards must decode back to exactly the same set.
+     *
+     * **Optional, like {@link maximum}.** A codec whose encoding has no size decision to make (a plain bitset has
+     * one representation and nothing to choose) simply omits it, and the engine skips the call.
+     *
+     * WHY THIS EXISTS. Roaring picks per container between an array, a bitset and a RUN — but the run choice is
+     * not automatic in any implementation: it is a `runOptimize()` pass you have to ask for, and nothing here was
+     * asking. So two of the three container types were ever used, and run-shaped data paid list or bitmap prices
+     * for a run. Measured on the shipped codec: a contiguous 1,000,000-id range serialized to 128.1 KiB where
+     * run-encoding needs **0.2 KiB** (570×), and a 2,000-run shape 536.5 KiB against **8.5 KiB** (63×). Sparse
+     * data is unchanged, because there are no runs to find — the pass is not a gamble.
+     *
+     * WHY ONLY ON THE COLD PATH. This is called where a whole immutable generation is written, so its cost
+     * amortizes over a write that is already serializing and checksumming every chunk. It is deliberately NOT
+     * called on the warm delta path (`chunk.ts`), which runs per operation: the hot path must not pay for a
+     * rare-ish win, per KISS/YAGNI. Warm rows are short-lived and get folded into a cold generation by
+     * compaction, where they are optimized then.
+     */
+    optimize?(): void;
 }
 /**
  * A pluggable bitmap codec — the factory the codec-agnostic engine constructs {@link CodecBitmap}s through.
@@ -2021,7 +2046,15 @@ interface DestroyResult {
      * `segment.erase` audit event keys off — a cleartext "erase" is not an irreversible destruction.
      */
     readonly cryptoShredded: boolean;
-    /** `'absent'` (no registry row), `'already'` (already destroyed), or `'cleartext'` (no DEK to shred). */
+    /**
+     * Why this segment was not a fresh crypto-shred. `'absent'` (no registry row), `'already'` (already
+     * destroyed), `'cleartext'` (no DEK to shred) — and from `eraseNamespace` only, where per-segment faults are
+     * isolated so one failure cannot discard the ledger: `'contended'` (warm rows were rewritten during every
+     * erase pass) or `` `failed: <message>` `` (any other fault).
+     *
+     * The last two come with `destroyed: false` and mean **the segment still holds data**. A namespace erase
+     * returns them rather than throwing, so its entries have to be inspected — see {@link eraseNamespace}.
+     */
     readonly reason?: string;
     /** Warm rows physically deleted as part of the erase. */
     readonly warmRowsDeleted: number;
@@ -2256,7 +2289,13 @@ declare class RetryingRegistryDriver implements IRegistryDriver {
 interface CrbmWriterOptions {
     /** Self-describing generation number (also encoded in the object key). */
     readonly generation: number;
-    readonly roaringSerializationId?: number;
+    /**
+     * Which codec produced the chunk payloads. Defaults to {@link PAYLOAD_CODEC_ROARING_PORTABLE}.
+     *
+     * Renamed from `roaringSerializationId` in 0.7.0 — same footer field, same offset and width, generalized
+     * because `.crbm` is a shared container and a second codec is expected before the format freezes.
+     */
+    readonly payloadCodecId?: number;
     readonly elementWidth?: number;
     /** When set, payloads + index are AES-256-GCM-encrypted, bound to `(segment, generation, scope)` via AAD. */
     readonly crypto?: CrbmCrypto;

package/dist/index.js

@@ -1394,8 +1394,8 @@ var FOOTER = {
   // u32
   flags: 20,
   // u32
-  roaringSerializationId: 24,
-  // u16
+  payloadCodecId: 24,
+  // u16 — was `roaringSerializationId`; same offset, same width, wider meaning
   elementWidth: 26,
   // u8
   containerCodec: 27,
@@ -1423,7 +1423,10 @@ var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
 var FLAG_ENCRYPTED = 1 << 0;
 var FLAG_LITTLE_ENDIAN = 1 << 2;
 var ELEMENT_WIDTH_32 = 32;
-var ROARING_PORTABLE_ID = 1;
+var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
+var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
+  PAYLOAD_CODEC_ROARING_PORTABLE
+]);
 var CONTAINER_CODEC_NONE = 0;
 var DEFAULT_TAIL_BYTES = 256 * 1024;
 var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
@@ -1520,10 +1523,10 @@ var CrbmReader = class _CrbmReader {
         `.crbm element_width ${elementWidth} not supported (v1 reads 32-bit ids; 64-bit is a future major version)`
       );
     }
-    const serializationId = fview.getUint16(FOOTER.roaringSerializationId, true);
-    if (serializationId !== ROARING_PORTABLE_ID) {
+    const payloadCodecId = fview.getUint16(FOOTER.payloadCodecId, true);
+    if (!KNOWN_PAYLOAD_CODEC_IDS.has(payloadCodecId)) {
       throw new UnsupportedError(
-        `.crbm roaring_serialization_id ${serializationId} not supported (v1 is portable=${ROARING_PORTABLE_ID})`
+        `.crbm payload_codec_id ${payloadCodecId} not supported by this build (known: ${[...KNOWN_PAYLOAD_CODEC_IDS].join(", ")}; ${PAYLOAD_CODEC_ROARING_PORTABLE}=roaring portable). A generation written by a different codec is rejected rather than decoded \u2014 a store uses one codec throughout.`
       );
     }
     const containerCodec = footer[FOOTER.containerCodec];
@@ -1818,8 +1821,8 @@ var CrbmWriter = class {
     view.setUint32(FOOTER.indexCrc32c, crc32c(indexRegion), true);
     view.setUint32(FOOTER.flags, FLAG_LITTLE_ENDIAN | (encrypted ? FLAG_ENCRYPTED : 0), true);
     view.setUint16(
-      FOOTER.roaringSerializationId,
-      this.options.roaringSerializationId ?? ROARING_PORTABLE_ID,
+      FOOTER.payloadCodecId,
+      this.options.payloadCodecId ?? PAYLOAD_CODEC_ROARING_PORTABLE,
       true
     );
     footer[FOOTER.elementWidth] = this.options.elementWidth ?? ELEMENT_WIDTH_32;
@@ -2074,6 +2077,7 @@ function writeCrbmGeneration(driver, key, chunks, options = {}) {
     const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
     for (const { chunkKey, bitmap } of sorted) {
       if (bitmap.isEmpty) continue;
+      bitmap.optimize?.();
       await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
       const pause = tick();
       if (pause !== null) await pause;
@@ -2089,6 +2093,7 @@ async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
     const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
     for await (const { chunkKey, bitmap } of chunks) {
       if (bitmap.isEmpty) continue;
+      bitmap.optimize?.();
       await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
       chunkKeys.push(chunkKey);
       cardinality += bitmap.size;
@@ -3671,7 +3676,22 @@ async function eraseNamespace(namespace, deps, options) {
   const destroyed = [];
   let segmentsShredded = 0;
   for (const ref of refs) {
-    const result = await shredSegment(ref, deps, options.allowCleartext ?? false);
+    let result;
+    try {
+      result = await shredSegment(ref, deps, options.allowCleartext ?? false);
+    } catch (err) {
+      result = {
+        segment: ref.segment,
+        namespace: ref.namespace,
+        destroyed: false,
+        cryptoShredded: false,
+        reason: isWriteConflictError(err) ? "contended" : `failed: ${err instanceof Error ? err.message : String(err)}`,
+        // Deliberately 0, and not a partial count. `eraseWarm` may have deleted rows before it threw, but the
+        // erase did not complete as a unit, so a partial figure on a failed entry would invite reading it as
+        // progress toward a destruction that did not happen. The attestation that matters is `destroyed`.
+        warmRowsDeleted: 0
+      };
+    }
     destroyed.push(result);
     if (result.cryptoShredded) {
       segmentsShredded += 1;