@cloudbitmaps/core 0.24.0 → 0.25.0

package/dist/core/codec.d.ts

@@ -16,9 +16,9 @@
  * different codecs, or a cached bitmap from codec A could reach codec B's in-place op. The `CloudRoaring` facade
  * mints the cache per store, so this cannot arise in normal use.)
  *
- * **Interface surface = exactly what the engine needs**, kept general enough for the known fast-follow codecs
- * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitset` will add live on that
- * package's own extended value type, not here — the engine never calls them). The one codec-linked concern
+ * **Interface surface = exactly what the engine needs**, and nothing a codec might one day offer: an operation the
+ * engine never calls (positional or rank/select access, say) is not on {@link CodecBitmap}, and a second codec that
+ * has such operations would carry them on its own value type. The one codec-linked concern
  * this seam intentionally leaves in the format layer is the `.crbm` **payload codec id**
  * (`PAYLOAD_CODEC_ROARING_PORTABLE` / `KNOWN_PAYLOAD_CODEC_IDS` in `crbm/format.ts`): it is stamped in the footer
  * and validated on read, as membership of a registry rather than equality with one constant, so a codec-specific

package/dist/core/cooperative.d.ts

@@ -22,7 +22,7 @@
  *    the microtask queue drains to empty before the event loop advances a single phase. A loop awaiting them
  *    yields to other *microtasks* and to nothing else; no pending I/O callback runs. Measured: 555 ms of
  *    starvation against a 568 ms unyielded baseline **on the synthetic yield-primitive benchmark below** —
- *    a different, smaller workload than the 442 ms end-to-end load above, which is why the two baselines
+ *    a different, smaller workload than the 518 ms end-to-end load above, which is why the two baselines
  *    differ. The yield must be a **macrotask** — hence
  *    `Clock.yieldNow`, backed by `setImmediate` in production wiring.
  * 2. **Yielding per unit of work costs more than the work.** The yield has to be periodic, and `everyN` has to be

package/dist/core/load-input.d.ts

@@ -115,10 +115,16 @@ export declare function prepareLoadInput(input: LoadInput, codec: CodecInterface
  * input of a load goes through, and what a flavor offers its users to decode bytes they hold themselves.
  *
  * The bytes must be a `Uint8Array` (a Node `Buffer` is one), at most {@link MAX_SERIALIZED_LOAD_BYTES}, one whole
- * bitmap and structurally sound. A view is read over the bytes it really holds, and bytes in a `SharedArrayBuffer`
- * are copied first. An empty or detached buffer is the empty bitmap.
+ * bitmap and structurally sound. A view is read over the bytes it really holds, once, and the size checks, the
+ * copy and the decode all use that one view. Bytes in a `SharedArrayBuffer` are copied first. An empty or detached
+ * buffer is the empty bitmap, unless `refuseEmpty` is set.
  *
  * @param what names the input in a refusal's message.
+ * @param options `maxBytes` lowers the size cap for this call (it cannot raise it); `refuseEmpty` refuses an empty or
+ * detached buffer, which is not a bitmap.
  * @throws {ValidationError} for anything that is not such bytes; nothing reaches the codec's decoder then.
  */
-export declare function decodeSerialized(bytes: unknown, codec: CodecInterface, what?: string): CodecBitmap;
+export declare function decodeSerialized(bytes: unknown, codec: CodecInterface, what?: string, options?: {
+    readonly maxBytes?: number;
+    readonly refuseEmpty?: boolean;
+}): CodecBitmap;

package/dist/index.d.ts

@@ -24,7 +24,7 @@ export { BoundedLru } from './core/lru.js';
 export { safeMetrics } from './core/metrics.js';
 export { runExport } from './export/index.js';
 export type { SegmentReader } from './export/index.js';
-export { splitId } from './core/bit-route.js';
+export { splitId, refusalOfId } from './core/bit-route.js';
 export { mapWithConcurrency } from './core/concurrency.js';
 export { resolveBudget, resolvePerOpBudget, checkBudget, collectWithinBudget } from './core/budget.js';
 export { segmentExists, listSegments } from './core/discover.js';
@@ -43,7 +43,7 @@ export { compileCombineMany, rebindCombineMany, runCombineMany } from './core/co
 export type { CombineManyDeps, CombineManyFeed, CombineManyOperand, CombineManyOutcome, CombineManyOutput, CombineManyRequest, CombineManyRun, CombineManyWrite, CompiledCombineMany, Expr, MaterializeManyFeedRecord, MaterializeManyOperandStats, MaterializeManyOutputStats, MaterializeManyStats, } from './core/combine-many.js';
 export type { LoadDeps, LoadOptions, LoadGuard, GuardRefusal, LoadJudgement, LoadResult, LoadRefusal, } from './core/load.js';
 export type { LoadInput, PortableBitmap } from './core/load-input.js';
-export { decodeSerialized } from './core/load-input.js';
+export { decodeSerialized, MAX_SERIALIZED_LOAD_BYTES } from './core/load-input.js';
 export { listGenerations, rollbackSegment } from './core/rollback.js';
 export type { GenerationListDeps, GenerationEntry, RollbackResult } from './core/rollback.js';
 export { eraseIdFromSegment, findIdUnderTombstone } from './core/erase-id.js';

package/dist/index.js

@@ -183,7 +183,7 @@ async function refuseAbsent(op, candidates, limit, exists, nameOf) {
 // src/core/chunk-checks.ts
 function assertChunkKeyInRange(k) {
   if (!Number.isInteger(k) || k < 0 || k >= CHUNK_COUNT) {
-    throw new IntegrityError(`chunk key from a tier is out of range: ${k}`);
+    throw new IntegrityError("a chunk key from a tier is out of range");
   }
 }
 function assertChunkCardinalityInRange(c) {
@@ -206,22 +206,22 @@ function checkedChunkKeys(listed2) {
   keys.sort((a, b) => a - b);
   for (let i = 1; i < keys.length; i++) {
     if (keys[i] === keys[i - 1]) {
-      throw new IntegrityError(`chunk key from a tier is listed twice: ${keys[i]}`);
+      throw new IntegrityError("a chunk key from a tier is listed twice");
     }
   }
   return keys;
 }
-function assertChunkPayloadInRange(bitmap, chunkKey) {
+function assertChunkPayloadInRange(bitmap) {
   const max = bitmap.maximum?.();
   if (max !== void 0 && max > MAX_REMAINDER) {
     throw new IntegrityError(
-      `chunk ${chunkKey} payload holds value ${max}, outside the 16-bit remainder range [0, ${MAX_REMAINDER}] \u2014 the stored object is corrupt or was not written by this codec`
+      `a chunk payload holds a value outside the 16-bit remainder range [0, ${MAX_REMAINDER}] (a container key other than 0) \u2014 the stored object is corrupt or was not written by this codec`
     );
   }
 }
-function decodeChunkBytes(codec, bytes, chunkKey, maxBitmapBytes) {
+function decodeChunkBytes(codec, bytes, maxBitmapBytes) {
   const bitmap = codec.safeDeserialize(bytes, maxBitmapBytes);
-  assertChunkPayloadInRange(bitmap, chunkKey);
+  assertChunkPayloadInRange(bitmap);
   return bitmap;
 }
 
@@ -238,12 +238,10 @@ var ChunkStream = class {
     return this.iterator.next().then(
       (step) => {
         if (step.done === true) {
-          throw this.failure?.error ?? new IntegrityError(`chunk stream ended before chunk ${key}`);
+          throw this.failure?.error ?? new IntegrityError("chunk stream ended before a chunk it was opened over");
         }
         if (step.value.key !== key) {
-          throw new IntegrityError(
-            `chunk stream answered chunk ${step.value.key} where ${key} was due`
-          );
+          throw new IntegrityError("chunk stream answered a chunk other than the one due");
         }
         return step.value;
       },
@@ -1201,9 +1199,7 @@ var SegmentEngine = class _SegmentEngine {
     const expected = counts.get(chunkKey);
     if (chunk === null || chunk.size !== expected) {
       const got = chunk === null ? "no bytes" : `${chunk.size} ids`;
-      throw new IntegrityError(
-        `chunk ${chunkKey} holds ${got}, but the object's index says ${expected}`
-      );
+      throw new IntegrityError(`a chunk holds ${got}, but the object's index says ${expected}`);
     }
     return chunk.toUint32Array ? chunk.toUint32Array() : Uint32Array.from(chunk);
   }
@@ -1871,7 +1867,7 @@ var SegmentEngine = class _SegmentEngine {
         streamed.moves.moved = true;
       }
       if (read.bytes === null) return null;
-      const bitmap = decodeChunkBytes(this.codec, read.bytes, chunkKey, this.maxBitmapBytes);
+      const bitmap = decodeChunkBytes(this.codec, read.bytes, this.maxBitmapBytes);
       if (this.cache && !streamed.invalidated && !this.scope.invalidatedSince(streamed.prefix, streamed.openedAt)) {
         const version = this.streamedVersion(streamed.gen, read.version);
         if (version !== null) {
@@ -2084,7 +2080,7 @@ var SegmentEngine = class _SegmentEngine {
       }
     }
     if (!bytes) return null;
-    const bitmap = decodeChunkBytes(this.codec, bytes, ref.chunkKey, this.maxBitmapBytes);
+    const bitmap = decodeChunkBytes(this.codec, bytes, this.maxBitmapBytes);
     if (key !== "" && this.scope.isOpen(cacheKey, token)) this.cache?.set(key, bitmap);
     return bitmap;
   }
@@ -2545,7 +2541,7 @@ function* planChunkReads(extents, region) {
   for (const extent of extents) {
     const extentEnd = extent.offset + extent.length;
     if (!(Number.isSafeInteger(extent.offset) && Number.isSafeInteger(extent.length) && extent.length >= 1 && extent.offset >= previousEnd && extentEnd <= region.end)) {
-      throw crbmIntegrity(`chunk ${extent.key} is not inside the chunk region, in order`);
+      throw crbmIntegrity("a chunk is not inside the chunk region, in order");
     }
     previousEnd = extentEnd;
     if (chunks.length > 0 && extent.offset - end <= MAX_COALESCE_GAP_BYTES && extentEnd - start <= MAX_COALESCED_READ_BYTES) {
@@ -2569,7 +2565,7 @@ function* planChunkReads(extents, region) {
 var U32_CEIL = U32_MAX + 1;
 function varintLength(value) {
   if (!Number.isInteger(value) || value < 0 || value >= U32_CEIL) {
-    throw new RangeError(`varint value must be a u32; got ${value}`);
+    throw new RangeError("varint value must be a u32");
   }
   if (value < 128) return 1;
   if (value < 16384) return 2;
@@ -3016,7 +3012,7 @@ var CrbmReader = class _CrbmReader {
     const offset = this.index.offsets[slot];
     const length = this.index.lengths[slot];
     if (offset < PAYLOAD_START || offset + length > this.objectSize - FOOTER_BYTES) {
-      throw crbmIntegrity(`chunk ${chunkKey} payload out of bounds`);
+      throw crbmIntegrity(`chunk at position ${slot} payload out of bounds`);
     }
     const held = this.keptRange(offset, length);
     return this.openChunk(slot, chunkKey, held ?? await this.blob.getRange(offset, length));
@@ -3043,11 +3039,11 @@ var CrbmReader = class _CrbmReader {
    */
   openChunk(slot, chunkKey, bytes) {
     if (crc32c(bytes) !== this.index.crcs[slot]) {
-      throw crbmIntegrity(`chunk ${chunkKey} payload CRC mismatch`);
+      throw crbmIntegrity(`chunk at position ${slot} payload CRC mismatch`);
     }
     if (this.crypto === void 0) return bytes;
     if (bytes.length < AEAD_NONCE_BYTES + AEAD_TAG_BYTES) {
-      throw crbmIntegrity(`chunk ${chunkKey} encrypted payload too short`);
+      throw crbmIntegrity(`chunk at position ${slot} encrypted payload too short`);
     }
     return this.crypto.aead.open(
       {
@@ -3293,19 +3289,20 @@ function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1
     const chunkKey = prevKey + keyDelta;
     const offset = prevEnd + offDelta;
     if (i > 0 && keyDelta === 0) {
-      throw crbmIntegrity(`.crbm index has a duplicate chunkKey ${chunkKey}`);
+      throw crbmIntegrity(`.crbm index entry ${i} repeats a chunk key`);
     }
-    if (chunkKey > 65535) throw crbmIntegrity(`.crbm chunkKey ${chunkKey} out of range`);
+    if (chunkKey > 65535)
+      throw crbmIntegrity(`.crbm index entry ${i} has a chunk key out of range`);
     if (len < minPayloadBytes || len > maxPayloadBytes) {
       throw crbmIntegrity(
-        `.crbm chunk ${chunkKey} length ${len} invalid (most is ${maxPayloadBytes})`
+        `.crbm index entry ${i} has a payload length ${len} invalid (most is ${maxPayloadBytes})`
       );
     }
     if (card < 1 || card > MAX_CHUNK_CARDINALITY) {
-      throw crbmIntegrity(`.crbm chunk ${chunkKey} cardinality ${card} invalid`);
+      throw crbmIntegrity(`.crbm index entry ${i} has a cardinality ${card} invalid`);
     }
     if (offset + len > payloadEnd) {
-      throw crbmIntegrity(`.crbm chunk ${chunkKey} payload out of bounds`);
+      throw crbmIntegrity(`.crbm index entry ${i} has a payload out of bounds`);
     }
     keys[count] = chunkKey;
     cardinalityMinusOne[count] = card - 1;
@@ -5070,11 +5067,11 @@ var PinnedStorageChunkSource = class {
   constructor(inner, pins) {
     this.inner = inner;
     this.pins = pins;
-    for (const [key, pin] of pins) {
+    for (const [pinKey, pin] of pins) {
       const held = pin;
       if (held.generation !== null && (typeof held.version !== "string" || typeof held.fingerprint !== "string"))
         throw new ValidationError(
-          `pin for "${key}" names generation ${held.generation} without its version and object fingerprint`
+          `pin for "${pinKey}" names generation ${held.generation} without its version and object fingerprint`
         );
     }
   }
@@ -5275,14 +5272,15 @@ function plainView(bytes, what) {
   }
   return length === 0 ? new Uint8Array(0) : new Uint8Array(buffer, offset, length);
 }
-var sharedByteLength = typeof SharedArrayBuffer === "function" ? Object.getOwnPropertyDescriptor(SharedArrayBuffer.prototype, "byteLength")?.get : void 0;
+var plainByteLength = Object.getOwnPropertyDescriptor(ArrayBuffer.prototype, "byteLength")?.get;
+var hasSharedMemory = typeof SharedArrayBuffer === "function";
 function isShared(buffer) {
-  if (sharedByteLength === void 0) return false;
+  if (!hasSharedMemory) return false;
   try {
-    sharedByteLength.call(buffer);
-    return true;
-  } catch {
+    plainByteLength.call(buffer);
     return false;
+  } catch {
+    return true;
   }
 }
 function overCap(what, length) {
@@ -5323,11 +5321,22 @@ function prepareLoadInput(input, codec) {
     `a load takes ids (an iterable of integers), { serialized } or { bitmap }, as the only key; got an object with keys [${keys.join(", ")}]. Load options go in the next argument.`
   );
 }
-function decodeSerialized(bytes, codec, what = "the buffer") {
+function decodeSerialized(bytes, codec, what = "the buffer", options = {}) {
   if (typeof bytes !== "object" || bytes === null || tagOf(bytes) !== "Uint8Array") {
     throw new ValidationError(`${what} must be a Uint8Array of portable Roaring bytes`);
   }
   const view = plainView(bytes, what);
+  if (view.byteLength === 0 && options.refuseEmpty === true) {
+    throw new ValidationError(
+      `${what} is empty or detached: an empty buffer is not a bitmap, and an empty bitmap is built without bytes`
+    );
+  }
+  const { maxBytes } = options;
+  if (maxBytes !== void 0 && maxBytes < MAX_SERIALIZED_LOAD_BYTES && view.byteLength > maxBytes) {
+    throw new ValidationError(
+      `${what} is ${view.byteLength} bytes, more than the ${maxBytes} this call allows (maxBytes)`
+    );
+  }
   if (view.byteLength > MAX_SERIALIZED_LOAD_BYTES) throw overCap(what, view.byteLength);
   const own = isShared(view.buffer) ? new Uint8Array(view) : view;
   try {
@@ -5465,12 +5474,10 @@ var CrbmWriter = class {
   async addChunk(chunkKey, payload, cardinality) {
     if (this.finished) throw new ValidationError("CrbmWriter already finished");
     if (!Number.isInteger(chunkKey) || chunkKey < 0 || chunkKey > 65535) {
-      throw new ValidationError(`chunkKey must be a u16; got ${chunkKey}`);
+      throw new ValidationError("chunkKey must be a u16");
     }
     if (chunkKey <= this.lastKey) {
-      throw new ValidationError(
-        `chunkKey ${chunkKey} not strictly ascending (last ${this.lastKey})`
-      );
+      throw new ValidationError("chunkKey is not strictly ascending");
     }
     if (!Number.isInteger(cardinality) || cardinality < 1 || cardinality > MAX_CHUNK_CARDINALITY) {
       throw new ValidationError(
@@ -6280,7 +6287,7 @@ async function collectChunks(input, codec, clock) {
       throw new ValidationError("a chunk's key must be an integer in [0, 65535]");
     }
     if (chunkKey <= last) {
-      throw new ValidationError(`chunk ${chunkKey} does not ascend: the last was ${last}`);
+      throw new ValidationError("a chunk's key does not ascend past the previous chunk's");
     }
     last = chunkKey;
     const max = bitmap.maximum?.();
@@ -8671,7 +8678,7 @@ var Run = class {
                 failOperand(
                   c,
                   new IntegrityError(
-                    `operand "${c.op.spec.name}" lists chunk ${key} but its generation holds no bytes for it`
+                    `operand "${c.op.spec.name}" lists a chunk its generation holds no bytes for`
                   ),
                   victims ?? wanting(c)
                 );
@@ -8680,13 +8687,10 @@ var Run = class {
               try {
                 const resident = residentBytes(bytes.length);
                 if (!this.ledger.reserve(resident, makeRoom)) {
-                  throw this.refuse(resident, `operand "${c.op.spec.name}" at chunk ${key}`);
+                  throw this.refuse(resident, `operand "${c.op.spec.name}" at one of its chunks`);
                 }
                 held += resident;
-                decoded.set(
-                  c.op.index,
-                  decodeChunkBytes(this.codec, bytes, key, this.maxBitmapBytes)
-                );
+                decoded.set(c.op.index, decodeChunkBytes(this.codec, bytes, this.maxBitmapBytes));
               } catch (error) {
                 failOperand(c, error, victims ?? wanting(c));
               }
@@ -9593,7 +9597,7 @@ async function holds(c, generation) {
       }
       throw err;
     }
-    return bytes !== null && codec.safeDeserialize(bytes, maxBytes).has(remainder);
+    return bytes !== null && decodeChunkBytes(codec, bytes, maxBytes).has(remainder);
   } catch (err) {
     if (isNotFoundError(err)) return null;
     throw err;
@@ -10004,8 +10008,7 @@ async function stage(c) {
     c.fromFingerprint = reader.fingerprint;
     const bytes = await read(() => reader.getChunk(chunkKey));
     if (bytes === null) return await notInCurrent(c);
-    const target = codec.safeDeserialize(bytes, maxBytes);
-    assertRemaindersInRange(target, chunkKey);
+    const target = decodeChunkBytes(codec, bytes, maxBytes);
     if (!target.has(remainder)) return await notInCurrent(c);
     target.remove(remainder);
     assertRegistryCanWrite(deps.registry, "eraseIdFromSegment");
@@ -10291,26 +10294,17 @@ async function* rewrite(reader, chunkKey, replacement, codec, maxBytes, read) {
       }
       const item = await stream.next();
       if (item.done === true || item.value.key !== k) {
-        throw new IntegrityError(`the chunk stream ended before chunk ${k}`);
+        throw new IntegrityError("the chunk stream ended before a chunk the rewrite needs");
       }
       const bytes = item.value.bytes;
       if (bytes === null) continue;
-      const bitmap = codec.safeDeserialize(bytes, maxBytes);
-      assertRemaindersInRange(bitmap, k);
+      const bitmap = decodeChunkBytes(codec, bytes, maxBytes);
       yield { chunkKey: k, bitmap };
     }
   } finally {
     await stream.return(void 0);
   }
 }
-function assertRemaindersInRange(bitmap, chunkKey) {
-  const max = bitmap.maximum?.();
-  if (max !== void 0 && max > MAX_REMAINDER) {
-    throw new IntegrityError(
-      `chunk ${chunkKey} payload holds value ${max}, outside the 16-bit remainder range [0, ${MAX_REMAINDER}] \u2014 the stored object is corrupt or was not written by this codec; refusing to carry it into a new generation`
-    );
-  }
-}
 
 // src/core/cas-retry.ts
 async function casRetry(deps, ref, plan, step) {
@@ -12104,6 +12098,7 @@ export {
   LeaseLimitError,
   MAX_LEASES_PER_SEGMENT,
   MAX_LEASE_MS,
+  MAX_SERIALIZED_LOAD_BYTES,
   MIN_EXPIRES_AT_MS,
   NOOP_METRICS,
   NotFoundError,
@@ -12139,6 +12134,7 @@ export {
   mapWithConcurrency,
   readRetentionPolicy,
   rebindCombineMany,
+  refusalOfId,
   refuseUnknownRefKeys,
   releaseLease,
   resolveBudget,