@cloudbitmaps/core 0.1.2 → 0.1.3

package/dist/index.d.cts

@@ -1,5 +1,5 @@
-import { a as IWarmDriver, h as ColdChunkSource, S as SegmentRef, W as WarmReadOptions, i as SegmentSize, I as IColdDriver, d as IRegistryDriver, j as IKeystore, b as ChunkRef, C as ColdCaps, G as GenKey, B as BlobSink, R as RegCaps, e as RegistryRecord, f as NewRegistryRecord, T as Token, g as RegistryPatch, c as WarmRow, N as NoRow, k as CrbmCrypto, l as BlobReader, m as WrappedDek, A as Aead, n as AeadSealed } from './ports-D3BrJ6ax.cjs';
-export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s as RegistryStatus, t as aadFor } from './ports-D3BrJ6ax.cjs';
+import { a as IWarmDriver, h as ColdChunkSource, S as SegmentRef, W as WarmReadOptions, i as SegmentSize, I as IColdDriver, d as IRegistryDriver, j as IKeystore, b as ChunkRef, C as ColdCaps, G as GenKey, B as BlobSink, R as RegCaps, e as RegistryRecord, f as NewRegistryRecord, T as Token, g as RegistryPatch, c as WarmRow, N as NoRow, k as CrbmCrypto, l as BlobReader, m as WrappedDek, A as Aead, n as AeadSealed } from './ports-5BQTGKze.cjs';
+export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s as RegistryStatus, t as aadFor } from './ports-5BQTGKze.cjs';
 
 /**
  * The **bitmap-codec seam**.
@@ -50,6 +50,17 @@ interface CodecBitmap {
     /** Ascending iterator over the set values. */
     [Symbol.iterator](): IterableIterator<number>;
     toArray(): number[];
+    /**
+     * The largest value in the set, or `undefined` when empty.
+     *
+     * **Optional, and deliberately so.** The engine uses it for one thing: asserting that a payload it is about
+     * to interpret **as a chunk** holds only 16-bit remainders (see {@link assertChunkPayload}). A codec that
+     * cannot answer this in better than O(n) should simply omit it — the engine then skips the check rather
+     * than walking every value on the read path, which is the one thing this must never cost.
+     *
+     * Roaring answers it in O(1) from its container index, so the flagship codec implements it.
+     */
+    maximum?(): number | undefined;
 }
 /**
  * A pluggable bitmap codec — the factory the codec-agnostic engine constructs {@link CodecBitmap}s through.
@@ -426,7 +437,7 @@ interface EngineDeps {
     readonly cache?: BoundedLru<string, CodecBitmap>;
     readonly maxBitmapBytes?: number;
     /**
-     * The bitmap codec (DECISIONS #58) — **required**. `core/` is
+     * The bitmap codec — **required**. `core/` is
      * codec-agnostic: it can have no default, because the concrete codec lives in a *flavor* package that depends
      * on core (a default here would invert that arrow). A flavor's facade injects it — `@cloudbitmaps/roaring`
      * passes `roaringCodec` — so applications never see this.
@@ -545,6 +556,21 @@ declare class SegmentEngine {
      * letting it flow into `joinId` and produce a bogus id.
      */
     private assertChunkKeyInRange;
+    /**
+     * The other half of invariant 5, and the one that was missing: a chunk payload holds **remainders** — 16-bit
+     * offsets within one chunk — so every value must be `<= 0xffff`.
+     *
+     * Nothing upstream established that. The byte cap bounds *size*, and CRC/AEAD prove the bytes are the bytes
+     * that were written — which anyone able to write the tier satisfies trivially. A value `>= 65536` then
+     * reaches `joinId`, which masks it (`remainder & 0xffff`) and emits a **fabricated id belonging to a
+     * different chunk's id space**: indistinguishable from real data, inflating `count()` and creating spurious
+     * `intersect` matches. Compaction is the only path that already failed loud (`CrbmWriter` rejects
+     * `cardinality > 65536`), which merely turns the same row into a permanently poison segment.
+     *
+     * Costs one `maximum()` per chunk, not per id — `maximum` is optional on the codec seam precisely so a codec
+     * that cannot answer in O(1) opts out instead of making the read path walk every value.
+     */
+    private assertChunkPayloadInRange;
     private effectiveChunk;
     /** The segment's current generation, resolved once per op (`undefined` ⇒ source can't report it). */
     private currentGen;
@@ -824,7 +850,7 @@ declare class RecordingAuditSink implements IAuditSink {
  * **Correctness without a lock:** the commit is the conditional `currentGen` swap; write-once generations +
  * version-fenced purge make even two concurrent compactions safe (the loser's staged generation is an orphan
  * GC'd later; no torn read — readers are pinned to a committed generation, **I5**). The lease only avoids that
- * wasted duplicate work (see DECISIONS #18).
+ * wasted duplicate work.
  *
  * Pure orchestration over the injected driver interfaces + an injected {@link Clock} (timer-free; the daemon
  * layer owns scheduling/heartbeat). A crash at *any* step is recoverable: nothing is destructive until after
@@ -839,7 +865,7 @@ interface CompactionDeps {
     readonly clock: Pick<Clock, 'now'>;
     readonly maxBitmapBytes?: number;
     /**
-     * Bitmap codec ([DECISIONS #58]) — compaction only merges bitmaps through it. Optional in the type so this
+     * Bitmap codec — compaction only merges bitmaps through it. Optional in the type so this
      * stays call-compatible public API; a **flavor** package binds it (see {@link requireCodec}).
      */
     readonly codec?: CodecInterface;
@@ -1040,7 +1066,7 @@ interface ExportOptions {
     /** Output format; defaults to `'roaring'`. */
     readonly format?: ExportFormat;
     /**
-     * Bitmap codec used to build the exported bitmap for the `'roaring'` format ([DECISIONS #58]). Optional in the
+     * Bitmap codec used to build the exported bitmap for the `'roaring'` format. Optional in the
      * type so this stays call-compatible public API; a **flavor** package binds it (see `requireCodec`). Not needed
      * for `'ndjson'`, which streams plain ids.
      */
@@ -1123,11 +1149,6 @@ interface BitRoute {
 }
 /** Split a u32 id into `{ chunkKey, remainder }`. Throws `ValidationError` for non-u32 input. */
 declare function splitId(id: number): BitRoute;
-/**
- * Recombine a chunk key + remainder back into the u32 id. Precondition: `chunkKey` is a validated
- * 16-bit value (tier-derived keys are validated upstream in the engine before reaching here, per
- * invariant 5); `remainder` is masked to 16 bits.
- */
 declare function joinId(chunkKey: number, remainder: number): number;
 
 /**
@@ -1516,14 +1537,14 @@ interface BulkLoadResult {
     readonly cardinality: number;
 }
 /**
- * **Bulk-load** a whole immutable generation from a flat id source — the batch "seed/sweep" entry point.
+ * **Bulk-load** a whole immutable generation from a flat id source — the batch "seed/rebuild" entry point.
  * Unlike {@link writeCrbmGeneration} (which takes pre-grouped bitmaps), this
  * consumes an arbitrary, **unsorted** stream of u32 ids (sync or async) and routes each into its chunk's
  * Roaring bitmap as it goes. The input is consumed lazily and ids dedup on insert, so memory is bounded by
  * the **built generation** — the in-memory Roaring representation of the *distinct* set (≤ one bitmap per
  * non-empty chunk) — not by the input length: you can stream a billion duplicate-heavy ids and hold only the
  * distinct result. Note this is `O(distinct set)`, **not** window-bounded like {@link SegmentEngine.intersect}:
- * bulk-load holds the whole generation in RAM, which suits batch seed/sweep jobs but not an
+ * bulk-load holds the whole generation in RAM, which suits batch seed/rebuild jobs but not an
  * unbounded-cardinality stream (a segment larger than RAM needs external-merge / pre-sorted input — later phase).
  *
  * Each id must be an integer in `[0, 2³²)` ({@link splitId} throws {@link ValidationError} otherwise) — the
@@ -1541,7 +1562,7 @@ declare function bulkLoadCrbmGeneration(driver: IColdDriver, key: GenKey, ids: I
     keystore?: IKeystore;
     requireEncryption?: boolean;
     audit?: IAuditSink;
-    /** Bitmap codec ([DECISIONS #58]). Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
+    /** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
     codec?: CodecInterface;
 }): Promise<BulkLoadResult>;
 
@@ -1696,7 +1717,7 @@ declare class InProcessKeystore implements IKeystore {
 }
 
 /**
- * Crypto-shred erasure (Phase 4e, [DECISIONS #19]).
+ * Crypto-shred erasure (Phase 4e).
  *
  * GDPR "right to erasure" on immutable/backed-up Cold storage: you can't delete a `.crbm` object from every
  * backup, but you can **delete its key**. `destroySegment` drops the segment's wrapped DEK(s) from the registry

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { a as IWarmDriver, h as ColdChunkSource, S as SegmentRef, W as WarmReadOptions, i as SegmentSize, I as IColdDriver, d as IRegistryDriver, j as IKeystore, b as ChunkRef, C as ColdCaps, G as GenKey, B as BlobSink, R as RegCaps, e as RegistryRecord, f as NewRegistryRecord, T as Token, g as RegistryPatch, c as WarmRow, N as NoRow, k as CrbmCrypto, l as BlobReader, m as WrappedDek, A as Aead, n as AeadSealed } from './ports-D3BrJ6ax.js';
-export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s as RegistryStatus, t as aadFor } from './ports-D3BrJ6ax.js';
+import { a as IWarmDriver, h as ColdChunkSource, S as SegmentRef, W as WarmReadOptions, i as SegmentSize, I as IColdDriver, d as IRegistryDriver, j as IKeystore, b as ChunkRef, C as ColdCaps, G as GenKey, B as BlobSink, R as RegCaps, e as RegistryRecord, f as NewRegistryRecord, T as Token, g as RegistryPatch, c as WarmRow, N as NoRow, k as CrbmCrypto, l as BlobReader, m as WrappedDek, A as Aead, n as AeadSealed } from './ports-5BQTGKze.js';
+export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s as RegistryStatus, t as aadFor } from './ports-5BQTGKze.js';
 
 /**
  * The **bitmap-codec seam**.
@@ -50,6 +50,17 @@ interface CodecBitmap {
     /** Ascending iterator over the set values. */
     [Symbol.iterator](): IterableIterator<number>;
     toArray(): number[];
+    /**
+     * The largest value in the set, or `undefined` when empty.
+     *
+     * **Optional, and deliberately so.** The engine uses it for one thing: asserting that a payload it is about
+     * to interpret **as a chunk** holds only 16-bit remainders (see {@link assertChunkPayload}). A codec that
+     * cannot answer this in better than O(n) should simply omit it — the engine then skips the check rather
+     * than walking every value on the read path, which is the one thing this must never cost.
+     *
+     * Roaring answers it in O(1) from its container index, so the flagship codec implements it.
+     */
+    maximum?(): number | undefined;
 }
 /**
  * A pluggable bitmap codec — the factory the codec-agnostic engine constructs {@link CodecBitmap}s through.
@@ -426,7 +437,7 @@ interface EngineDeps {
     readonly cache?: BoundedLru<string, CodecBitmap>;
     readonly maxBitmapBytes?: number;
     /**
-     * The bitmap codec (DECISIONS #58) — **required**. `core/` is
+     * The bitmap codec — **required**. `core/` is
      * codec-agnostic: it can have no default, because the concrete codec lives in a *flavor* package that depends
      * on core (a default here would invert that arrow). A flavor's facade injects it — `@cloudbitmaps/roaring`
      * passes `roaringCodec` — so applications never see this.
@@ -545,6 +556,21 @@ declare class SegmentEngine {
      * letting it flow into `joinId` and produce a bogus id.
      */
     private assertChunkKeyInRange;
+    /**
+     * The other half of invariant 5, and the one that was missing: a chunk payload holds **remainders** — 16-bit
+     * offsets within one chunk — so every value must be `<= 0xffff`.
+     *
+     * Nothing upstream established that. The byte cap bounds *size*, and CRC/AEAD prove the bytes are the bytes
+     * that were written — which anyone able to write the tier satisfies trivially. A value `>= 65536` then
+     * reaches `joinId`, which masks it (`remainder & 0xffff`) and emits a **fabricated id belonging to a
+     * different chunk's id space**: indistinguishable from real data, inflating `count()` and creating spurious
+     * `intersect` matches. Compaction is the only path that already failed loud (`CrbmWriter` rejects
+     * `cardinality > 65536`), which merely turns the same row into a permanently poison segment.
+     *
+     * Costs one `maximum()` per chunk, not per id — `maximum` is optional on the codec seam precisely so a codec
+     * that cannot answer in O(1) opts out instead of making the read path walk every value.
+     */
+    private assertChunkPayloadInRange;
     private effectiveChunk;
     /** The segment's current generation, resolved once per op (`undefined` ⇒ source can't report it). */
     private currentGen;
@@ -824,7 +850,7 @@ declare class RecordingAuditSink implements IAuditSink {
  * **Correctness without a lock:** the commit is the conditional `currentGen` swap; write-once generations +
  * version-fenced purge make even two concurrent compactions safe (the loser's staged generation is an orphan
  * GC'd later; no torn read — readers are pinned to a committed generation, **I5**). The lease only avoids that
- * wasted duplicate work (see DECISIONS #18).
+ * wasted duplicate work.
  *
  * Pure orchestration over the injected driver interfaces + an injected {@link Clock} (timer-free; the daemon
  * layer owns scheduling/heartbeat). A crash at *any* step is recoverable: nothing is destructive until after
@@ -839,7 +865,7 @@ interface CompactionDeps {
     readonly clock: Pick<Clock, 'now'>;
     readonly maxBitmapBytes?: number;
     /**
-     * Bitmap codec ([DECISIONS #58]) — compaction only merges bitmaps through it. Optional in the type so this
+     * Bitmap codec — compaction only merges bitmaps through it. Optional in the type so this
      * stays call-compatible public API; a **flavor** package binds it (see {@link requireCodec}).
      */
     readonly codec?: CodecInterface;
@@ -1040,7 +1066,7 @@ interface ExportOptions {
     /** Output format; defaults to `'roaring'`. */
     readonly format?: ExportFormat;
     /**
-     * Bitmap codec used to build the exported bitmap for the `'roaring'` format ([DECISIONS #58]). Optional in the
+     * Bitmap codec used to build the exported bitmap for the `'roaring'` format. Optional in the
      * type so this stays call-compatible public API; a **flavor** package binds it (see `requireCodec`). Not needed
      * for `'ndjson'`, which streams plain ids.
      */
@@ -1123,11 +1149,6 @@ interface BitRoute {
 }
 /** Split a u32 id into `{ chunkKey, remainder }`. Throws `ValidationError` for non-u32 input. */
 declare function splitId(id: number): BitRoute;
-/**
- * Recombine a chunk key + remainder back into the u32 id. Precondition: `chunkKey` is a validated
- * 16-bit value (tier-derived keys are validated upstream in the engine before reaching here, per
- * invariant 5); `remainder` is masked to 16 bits.
- */
 declare function joinId(chunkKey: number, remainder: number): number;
 
 /**
@@ -1516,14 +1537,14 @@ interface BulkLoadResult {
     readonly cardinality: number;
 }
 /**
- * **Bulk-load** a whole immutable generation from a flat id source — the batch "seed/sweep" entry point.
+ * **Bulk-load** a whole immutable generation from a flat id source — the batch "seed/rebuild" entry point.
  * Unlike {@link writeCrbmGeneration} (which takes pre-grouped bitmaps), this
  * consumes an arbitrary, **unsorted** stream of u32 ids (sync or async) and routes each into its chunk's
  * Roaring bitmap as it goes. The input is consumed lazily and ids dedup on insert, so memory is bounded by
  * the **built generation** — the in-memory Roaring representation of the *distinct* set (≤ one bitmap per
  * non-empty chunk) — not by the input length: you can stream a billion duplicate-heavy ids and hold only the
  * distinct result. Note this is `O(distinct set)`, **not** window-bounded like {@link SegmentEngine.intersect}:
- * bulk-load holds the whole generation in RAM, which suits batch seed/sweep jobs but not an
+ * bulk-load holds the whole generation in RAM, which suits batch seed/rebuild jobs but not an
  * unbounded-cardinality stream (a segment larger than RAM needs external-merge / pre-sorted input — later phase).
  *
  * Each id must be an integer in `[0, 2³²)` ({@link splitId} throws {@link ValidationError} otherwise) — the
@@ -1541,7 +1562,7 @@ declare function bulkLoadCrbmGeneration(driver: IColdDriver, key: GenKey, ids: I
     keystore?: IKeystore;
     requireEncryption?: boolean;
     audit?: IAuditSink;
-    /** Bitmap codec ([DECISIONS #58]). Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
+    /** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
     codec?: CodecInterface;
 }): Promise<BulkLoadResult>;
 
@@ -1696,7 +1717,7 @@ declare class InProcessKeystore implements IKeystore {
 }
 
 /**
- * Crypto-shred erasure (Phase 4e, [DECISIONS #19]).
+ * Crypto-shred erasure (Phase 4e).
  *
  * GDPR "right to erasure" on immutable/backed-up Cold storage: you can't delete a `.crbm` object from every
  * backup, but you can **delete its key**. `destroySegment` drops the segment's wrapped DEK(s) from the registry

package/dist/index.js

@@ -1,8 +1,8 @@
 import { mapWithConcurrency } from './chunk-SNJVZ227.js';
 export { mapWithConcurrency } from './chunk-SNJVZ227.js';
-import { validateNewRegistryRecord, recordFromNew, validateRegistryPatch, applyRegistryPatch, registryCounterOf, parseRegistryEnvelope, serializeRegistryEnvelope } from './chunk-2YDULGXS.js';
-import { NO_ROW } from './chunk-AS6ODRLT.js';
-export { NO_ROW } from './chunk-AS6ODRLT.js';
+import { validateNewRegistryRecord, recordFromNew, validateRegistryPatch, applyRegistryPatch, registryCounterOf, parseRegistryEnvelope, serializeRegistryEnvelope } from './chunk-L54FL5DW.js';
+import { NO_ROW } from './chunk-YEHMG6W2.js';
+export { NO_ROW } from './chunk-YEHMG6W2.js';
 import { ValidationError, BudgetExceededError, isTransientError, isWriteConflictError, WriteConflictError, IntegrityError, UnsupportedError, CapabilityError, KeyUnavailableError, validateChunkRef, validateSegmentRef, isNotFoundError, NotFoundError, TransientError, namespacePart } from './chunk-NUIDEEFZ.js';
 export { BudgetExceededError, CapabilityError, CloudRoaringError, IntegrityError, KeyUnavailableError, NotFoundError, TimeoutError, TransientError, UnsupportedError, ValidationError, WriteConflictError, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransientError, isValidationError, isWriteConflictError, validateSegmentRef } from './chunk-NUIDEEFZ.js';
 import { createHash, randomUUID, randomBytes, createCipheriv, createDecipheriv } from 'crypto';
@@ -20,6 +20,7 @@ function splitId(id) {
   }
   return { chunkKey: id >>> 16, remainder: id & LOW_16 };
 }
+var MAX_REMAINDER = CHUNK_COUNT - 1;
 function joinId(chunkKey, remainder) {
   return ((chunkKey & LOW_16) << 16 >>> 0) + (remainder & LOW_16);
 }
@@ -82,8 +83,18 @@ function decodeDelta(bytes, maxBitmapBytes, codec) {
   }
   const adds = codec.safeDeserialize(bytes.subarray(addsStart, addsEnd), maxBitmapBytes);
   const removes = codec.safeDeserialize(bytes.subarray(addsEnd), maxBitmapBytes);
+  assertRemainderRange(adds, "adds");
+  assertRemainderRange(removes, "removes");
   return { adds, removes };
 }
+function assertRemainderRange(bitmap, half) {
+  const max = bitmap.maximum?.();
+  if (max !== void 0 && max > MAX_REMAINDER) {
+    throw new IntegrityError(
+      `warm delta ${half} holds value ${max}, outside the 16-bit remainder range [0, ${MAX_REMAINDER}] \u2014 the stored row is corrupt or was not written by this codec`
+    );
+  }
+}
 
 // src/core/budget.ts
 var DEFAULT_BUDGET = { maxRequests: 1e6 };
@@ -611,6 +622,28 @@ var SegmentEngine = class {
       throw new IntegrityError(`chunk key from a tier is out of range: ${k}`);
     }
   }
+  /**
+   * The other half of invariant 5, and the one that was missing: a chunk payload holds **remainders** — 16-bit
+   * offsets within one chunk — so every value must be `<= 0xffff`.
+   *
+   * Nothing upstream established that. The byte cap bounds *size*, and CRC/AEAD prove the bytes are the bytes
+   * that were written — which anyone able to write the tier satisfies trivially. A value `>= 65536` then
+   * reaches `joinId`, which masks it (`remainder & 0xffff`) and emits a **fabricated id belonging to a
+   * different chunk's id space**: indistinguishable from real data, inflating `count()` and creating spurious
+   * `intersect` matches. Compaction is the only path that already failed loud (`CrbmWriter` rejects
+   * `cardinality > 65536`), which merely turns the same row into a permanently poison segment.
+   *
+   * Costs one `maximum()` per chunk, not per id — `maximum` is optional on the codec seam precisely so a codec
+   * that cannot answer in O(1) opts out instead of making the read path walk every value.
+   */
+  assertChunkPayloadInRange(bitmap, chunkKey) {
+    const max = bitmap.maximum?.();
+    if (max !== void 0 && max > MAX_REMAINDER) {
+      throw new IntegrityError(
+        `chunk ${chunkKey} payload holds value ${max}, outside the 16-bit remainder range [0, ${MAX_REMAINDER}] \u2014 the stored object is corrupt or was not written by this codec`
+      );
+    }
+  }
   async effectiveChunk(seg, chunkKey, warmRows, gen) {
     const warmBytes = warmRows.get(chunkKey);
     const delta = warmBytes ? decodeDelta(warmBytes, this.maxBitmapBytes, this.codec) : emptyDelta(this.codec);
@@ -655,6 +688,7 @@ var SegmentEngine = class {
     }
     if (!bytes) return null;
     const bitmap = this.codec.safeDeserialize(bytes, this.maxBitmapBytes);
+    this.assertChunkPayloadInRange(bitmap, ref.chunkKey);
     this.cache?.set(cacheKey, bitmap);
     return bitmap;
   }