@cloudbitmaps/core 0.4.1 → 0.6.0

package/dist/index.d.cts

@@ -7,7 +7,7 @@ export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s
  * `core/` is **codec-agnostic**: `SegmentEngine`, compaction, and the `.crbm` read/write helpers only ever
  * construct and combine bitmaps through the {@link CodecInterface} factory + the {@link CodecBitmap} value type
  * defined here — never a concrete implementation. The flagship codec is roaring (`roaringCodec`, today in
- * `core/bitmap.ts`; it moves to `@cloudbitmaps/roaring` when the package split lands); `@cloudbitmaps/bitmap`
+ * `core/bitmap.ts`; it moves to `@cloudbitmaps/roaring` when the package split lands); `@cloudbitmaps/bitset`
  * (plain bitset) and `@cloudbitmaps/soaring` plug in behind the same seam with zero engine or driver changes.
  *
  * **Homogeneity contract:** a single store uses a single codec, so every {@link CodecBitmap} an operation sees
@@ -18,7 +18,7 @@ export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s
  * mints the cache per store, so this cannot arise in normal use.)
  *
  * **Interface surface = exactly what the engine needs**, kept general enough for the known fast-follow codecs
- * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitmap` will add live on that
+ * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitset` will add live on that
  * package's own extended value type, not here — the engine never calls them). The one codec-linked concern
  * this seam intentionally leaves in the format layer is the `.crbm` **serialization id** (`ROARING_PORTABLE_ID`
  * in `crbm/format.ts`): it is stamped in the footer and validated on read, and generalizes to a per-codec id
@@ -61,6 +61,27 @@ interface CodecBitmap {
      * Roaring answers it in O(1) from its container index, so the flagship codec implements it.
      */
     maximum?(): number | undefined;
+    /**
+     * Re-encode for size, in place, immediately before a **cold** write. Representation only — this must never
+     * change membership, and `serialize()` afterwards must decode back to exactly the same set.
+     *
+     * **Optional, like {@link maximum}.** A codec whose encoding has no size decision to make (a plain bitset has
+     * one representation and nothing to choose) simply omits it, and the engine skips the call.
+     *
+     * WHY THIS EXISTS. Roaring picks per container between an array, a bitset and a RUN — but the run choice is
+     * not automatic in any implementation: it is a `runOptimize()` pass you have to ask for, and nothing here was
+     * asking. So two of the three container types were ever used, and run-shaped data paid list or bitmap prices
+     * for a run. Measured on the shipped codec: a contiguous 1,000,000-id range serialized to 128.1 KiB where
+     * run-encoding needs **0.2 KiB** (570×), and a 2,000-run shape 536.5 KiB against **8.5 KiB** (63×). Sparse
+     * data is unchanged, because there are no runs to find — the pass is not a gamble.
+     *
+     * WHY ONLY ON THE COLD PATH. This is called where a whole immutable generation is written, so its cost
+     * amortizes over a write that is already serializing and checksumming every chunk. It is deliberately NOT
+     * called on the warm delta path (`chunk.ts`), which runs per operation: the hot path must not pay for a
+     * rare-ish win, per KISS/YAGNI. Warm rows are short-lived and get folded into a cold generation by
+     * compaction, where they are optimized then.
+     */
+    optimize?(): void;
 }
 /**
  * A pluggable bitmap codec — the factory the codec-agnostic engine constructs {@link CodecBitmap}s through.
@@ -2021,7 +2042,15 @@ interface DestroyResult {
      * `segment.erase` audit event keys off — a cleartext "erase" is not an irreversible destruction.
      */
     readonly cryptoShredded: boolean;
-    /** `'absent'` (no registry row), `'already'` (already destroyed), or `'cleartext'` (no DEK to shred). */
+    /**
+     * Why this segment was not a fresh crypto-shred. `'absent'` (no registry row), `'already'` (already
+     * destroyed), `'cleartext'` (no DEK to shred) — and from `eraseNamespace` only, where per-segment faults are
+     * isolated so one failure cannot discard the ledger: `'contended'` (warm rows were rewritten during every
+     * erase pass) or `` `failed: <message>` `` (any other fault).
+     *
+     * The last two come with `destroyed: false` and mean **the segment still holds data**. A namespace erase
+     * returns them rather than throwing, so its entries have to be inspected — see {@link eraseNamespace}.
+     */
     readonly reason?: string;
     /** Warm rows physically deleted as part of the erase. */
     readonly warmRowsDeleted: number;
@@ -2131,11 +2160,23 @@ declare function runConsistencyCheck(deps: {
  * I/O of their own); the wrapped driver is responsible for *classifying* its transient faults (raising
  * {@link TransientError}); these decorators decide *whether and when* to retry.
  *
- * **Streaming methods** (`listChunks`, `list`) are retried by **re-enumerating from the start**, buffering
- * the result — a partially-consumed async iterator can't be safely resumed mid-stream (it would re-yield
- * earlier items). Their consumers in the engine already collect fully (the Warm dirty-set is small under
- * Topology-A; the cold `list` is a discovery scan), so buffering preserves current semantics. Point methods
- * are retried in place with no buffering.
+ * **Streaming methods split into two shapes**, and the difference is deliberate — a partially-consumed async
+ * iterator cannot be resumed mid-stream (it would re-yield earlier items), so each method picks between
+ * buffering and bounded memory:
+ *
+ *   · `RetryingColdDriver.list` / `RetryingRegistryDriver.list` **buffer**, inside `withRetry`, so the whole
+ *     enumeration is retried by re-running it from the start. Both are discovery scans over a generation or
+ *     namespace listing, where the result set is small and whole-scan retry is worth the memory.
+ *   · `RetryingWarmDriver.listChunks` does **not** buffer. It retries only until the first row arrives and
+ *     then streams live, because buffering it defeated the engine's resident-memory bound outright — see the
+ *     measured account on the method itself. A mid-stream fault therefore propagates rather than being
+ *     retried, which is the documented trade there.
+ *
+ * Point methods are retried in place, with nothing to buffer.
+ *
+ * (This paragraph previously said streaming methods buffer, full stop. That described `listChunks` as it was
+ * before the bound was fixed, and the stale wording had also been copied into the Postgres and MySQL warm
+ * drivers — so three comments promised a mid-stream resilience that the hot read path does not have.)
  *
  * What is **not** retried here: {@link WriteConflictError} (OCC — the engine's read-modify-write loop owns
  * that; a blind replay would re-apply against a stale token), and every deterministic error

package/dist/index.d.ts

@@ -7,7 +7,7 @@ export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s
  * `core/` is **codec-agnostic**: `SegmentEngine`, compaction, and the `.crbm` read/write helpers only ever
  * construct and combine bitmaps through the {@link CodecInterface} factory + the {@link CodecBitmap} value type
  * defined here — never a concrete implementation. The flagship codec is roaring (`roaringCodec`, today in
- * `core/bitmap.ts`; it moves to `@cloudbitmaps/roaring` when the package split lands); `@cloudbitmaps/bitmap`
+ * `core/bitmap.ts`; it moves to `@cloudbitmaps/roaring` when the package split lands); `@cloudbitmaps/bitset`
  * (plain bitset) and `@cloudbitmaps/soaring` plug in behind the same seam with zero engine or driver changes.
  *
  * **Homogeneity contract:** a single store uses a single codec, so every {@link CodecBitmap} an operation sees
@@ -18,7 +18,7 @@ export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s
  * mints the cache per store, so this cannot arise in normal use.)
  *
  * **Interface surface = exactly what the engine needs**, kept general enough for the known fast-follow codecs
- * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitmap` will add live on that
+ * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitset` will add live on that
  * package's own extended value type, not here — the engine never calls them). The one codec-linked concern
  * this seam intentionally leaves in the format layer is the `.crbm` **serialization id** (`ROARING_PORTABLE_ID`
  * in `crbm/format.ts`): it is stamped in the footer and validated on read, and generalizes to a per-codec id
@@ -61,6 +61,27 @@ interface CodecBitmap {
      * Roaring answers it in O(1) from its container index, so the flagship codec implements it.
      */
     maximum?(): number | undefined;
+    /**
+     * Re-encode for size, in place, immediately before a **cold** write. Representation only — this must never
+     * change membership, and `serialize()` afterwards must decode back to exactly the same set.
+     *
+     * **Optional, like {@link maximum}.** A codec whose encoding has no size decision to make (a plain bitset has
+     * one representation and nothing to choose) simply omits it, and the engine skips the call.
+     *
+     * WHY THIS EXISTS. Roaring picks per container between an array, a bitset and a RUN — but the run choice is
+     * not automatic in any implementation: it is a `runOptimize()` pass you have to ask for, and nothing here was
+     * asking. So two of the three container types were ever used, and run-shaped data paid list or bitmap prices
+     * for a run. Measured on the shipped codec: a contiguous 1,000,000-id range serialized to 128.1 KiB where
+     * run-encoding needs **0.2 KiB** (570×), and a 2,000-run shape 536.5 KiB against **8.5 KiB** (63×). Sparse
+     * data is unchanged, because there are no runs to find — the pass is not a gamble.
+     *
+     * WHY ONLY ON THE COLD PATH. This is called where a whole immutable generation is written, so its cost
+     * amortizes over a write that is already serializing and checksumming every chunk. It is deliberately NOT
+     * called on the warm delta path (`chunk.ts`), which runs per operation: the hot path must not pay for a
+     * rare-ish win, per KISS/YAGNI. Warm rows are short-lived and get folded into a cold generation by
+     * compaction, where they are optimized then.
+     */
+    optimize?(): void;
 }
 /**
  * A pluggable bitmap codec — the factory the codec-agnostic engine constructs {@link CodecBitmap}s through.
@@ -2021,7 +2042,15 @@ interface DestroyResult {
      * `segment.erase` audit event keys off — a cleartext "erase" is not an irreversible destruction.
      */
     readonly cryptoShredded: boolean;
-    /** `'absent'` (no registry row), `'already'` (already destroyed), or `'cleartext'` (no DEK to shred). */
+    /**
+     * Why this segment was not a fresh crypto-shred. `'absent'` (no registry row), `'already'` (already
+     * destroyed), `'cleartext'` (no DEK to shred) — and from `eraseNamespace` only, where per-segment faults are
+     * isolated so one failure cannot discard the ledger: `'contended'` (warm rows were rewritten during every
+     * erase pass) or `` `failed: <message>` `` (any other fault).
+     *
+     * The last two come with `destroyed: false` and mean **the segment still holds data**. A namespace erase
+     * returns them rather than throwing, so its entries have to be inspected — see {@link eraseNamespace}.
+     */
     readonly reason?: string;
     /** Warm rows physically deleted as part of the erase. */
     readonly warmRowsDeleted: number;
@@ -2131,11 +2160,23 @@ declare function runConsistencyCheck(deps: {
  * I/O of their own); the wrapped driver is responsible for *classifying* its transient faults (raising
  * {@link TransientError}); these decorators decide *whether and when* to retry.
  *
- * **Streaming methods** (`listChunks`, `list`) are retried by **re-enumerating from the start**, buffering
- * the result — a partially-consumed async iterator can't be safely resumed mid-stream (it would re-yield
- * earlier items). Their consumers in the engine already collect fully (the Warm dirty-set is small under
- * Topology-A; the cold `list` is a discovery scan), so buffering preserves current semantics. Point methods
- * are retried in place with no buffering.
+ * **Streaming methods split into two shapes**, and the difference is deliberate — a partially-consumed async
+ * iterator cannot be resumed mid-stream (it would re-yield earlier items), so each method picks between
+ * buffering and bounded memory:
+ *
+ *   · `RetryingColdDriver.list` / `RetryingRegistryDriver.list` **buffer**, inside `withRetry`, so the whole
+ *     enumeration is retried by re-running it from the start. Both are discovery scans over a generation or
+ *     namespace listing, where the result set is small and whole-scan retry is worth the memory.
+ *   · `RetryingWarmDriver.listChunks` does **not** buffer. It retries only until the first row arrives and
+ *     then streams live, because buffering it defeated the engine's resident-memory bound outright — see the
+ *     measured account on the method itself. A mid-stream fault therefore propagates rather than being
+ *     retried, which is the documented trade there.
+ *
+ * Point methods are retried in place, with nothing to buffer.
+ *
+ * (This paragraph previously said streaming methods buffer, full stop. That described `listChunks` as it was
+ * before the bound was fixed, and the stale wording had also been copied into the Postgres and MySQL warm
+ * drivers — so three comments promised a mid-stream resilience that the hot read path does not have.)
  *
  * What is **not** retried here: {@link WriteConflictError} (OCC — the engine's read-modify-write loop owns
  * that; a blind replay would re-apply against a stale token), and every deterministic error

package/dist/index.js

@@ -2074,6 +2074,7 @@ function writeCrbmGeneration(driver, key, chunks, options = {}) {
     const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
     for (const { chunkKey, bitmap } of sorted) {
       if (bitmap.isEmpty) continue;
+      bitmap.optimize?.();
       await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
       const pause = tick();
       if (pause !== null) await pause;
@@ -2089,6 +2090,7 @@ async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
     const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
     for await (const { chunkKey, bitmap } of chunks) {
       if (bitmap.isEmpty) continue;
+      bitmap.optimize?.();
       await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
       chunkKeys.push(chunkKey);
       cardinality += bitmap.size;
@@ -2108,6 +2110,10 @@ async function publishGeneration(registry, key, options = {}) {
           currentGen: key.generation,
           wrappedDeks: options.wrappedDeks
         });
+      } else if (record.status === "destroyed") {
+        throw new ValidationError(
+          `segment "${key.segment}" was destroyed (crypto-shredded) while generation ${key.generation} was being written \u2014 refusing to publish it; the written object is unreadable. Use a new segment.`
+        );
       } else if (record.currentGen > key.generation) {
         return false;
       } else if (record.currentGen === key.generation) {
@@ -3667,7 +3673,22 @@ async function eraseNamespace(namespace, deps, options) {
   const destroyed = [];
   let segmentsShredded = 0;
   for (const ref of refs) {
-    const result = await shredSegment(ref, deps, options.allowCleartext ?? false);
+    let result;
+    try {
+      result = await shredSegment(ref, deps, options.allowCleartext ?? false);
+    } catch (err) {
+      result = {
+        segment: ref.segment,
+        namespace: ref.namespace,
+        destroyed: false,
+        cryptoShredded: false,
+        reason: isWriteConflictError(err) ? "contended" : `failed: ${err instanceof Error ? err.message : String(err)}`,
+        // Deliberately 0, and not a partial count. `eraseWarm` may have deleted rows before it threw, but the
+        // erase did not complete as a unit, so a partial figure on a failed entry would invite reading it as
+        // progress toward a destruction that did not happen. The attestation that matters is `destroyed`.
+        warmRowsDeleted: 0
+      };
+    }
     destroyed.push(result);
     if (result.cryptoShredded) {
       segmentsShredded += 1;
@@ -3730,12 +3751,13 @@ async function shredSegment(ref, deps, allowCleartext) {
 }
 async function eraseWarm(warm, ref) {
   let deleted = 0;
+  let conflicts = 0;
   for (let pass = 0; pass < MAX_WARM_PASSES; pass++) {
+    conflicts = 0;
     const rows = [];
     for await (const row of warm.listChunks(ref))
       rows.push({ chunkKey: row.chunkKey, token: row.token });
     if (rows.length === 0) break;
-    let conflicts = 0;
     for (const { chunkKey, token } of rows) {
       const chunkRef = { namespace: ref.namespace, segment: ref.segment, chunkKey };
       try {
@@ -3748,6 +3770,11 @@ async function eraseWarm(warm, ref) {
     }
     if (conflicts === 0) break;
   }
+  if (conflicts > 0) {
+    throw new WriteConflictError(
+      `destroySegment: ${conflicts} warm row(s) of "${ref.segment}" were rewritten during every one of ${MAX_WARM_PASSES} erase passes \u2014 the segment was NOT destroyed and its cleartext warm deltas are still readable. Stop writes to this segment and retry.`
+    );
+  }
   return deleted;
 }
 
@@ -3878,11 +3905,15 @@ var RetryingWarmDriver = class {
       this.deps
     );
     if (started.first.done === true) return;
-    yield started.first.value;
-    for (; ; ) {
-      const next = await started.iterator.next();
-      if (next.done === true) return;
-      yield next.value;
+    try {
+      yield started.first.value;
+      for (; ; ) {
+        const next = await started.iterator.next();
+        if (next.done === true) return;
+        yield next.value;
+      }
+    } finally {
+      await started.iterator.return?.();
     }
   }
 };