@cloudbitmaps/core 0.6.0 → 0.8.0

package/dist/index.js

@@ -414,6 +414,55 @@ var SegmentEngine = class {
   removeMany(seg, ids) {
     return this.mutateMany(seg, ids, applyRemoveAll);
   }
+  /**
+   * **Atomically claim ids: add them, and return only the ones that were not already there.**
+   *
+   * The durable analogue of Redis `SETBIT` returning the prior bit — the primitive an exactly-once "have I
+   * already processed / already sent to this id?" check needs. `has()` then `add()` cannot do this: two workers
+   * can both read absent and both proceed.
+   *
+   * **Bulk on purpose.** The per-id shape is the one this library is worst at, because a Warm write rewrites a
+   * whole 64K-id chunk bitmap: claiming 5,000 ids one at a time costs ~5,000 writes and ~23 MB, and claiming
+   * them in one call costs 1 write and ~8 KB. So this takes a batch and does one OCC read-modify-write per
+   * distinct chunk — Redis's semantics at this library's cost shape, rather than Redis's cost shape.
+   *
+   * **Atomicity is per id, which is the guarantee that matters here.** Each id lives in exactly one chunk, and a
+   * chunk is one OCC row, so for any id exactly one concurrent claimer sees it as new. It is NOT atomic across
+   * chunks (neither is `addMany`) — a mid-flight failure can leave some chunks claimed. Re-running is safe:
+   * already-claimed ids simply come back as not-new.
+   *
+   * **The presence test consults Cold, not just the Warm delta.** Checking `adds` alone would report an id as
+   * newly claimed when a compaction had already folded it into a Cold generation — silently breaking
+   * exactly-once on any segment that has ever been compacted. The check is the full effective set,
+   * `(cold ∪ adds) \ removes`.
+   *
+   * Returns the claimed ids grouped nowhere in particular — order is unspecified.
+   */
+  async claimMany(seg, ids) {
+    const byChunk = /* @__PURE__ */ new Map();
+    for (const id of ids) {
+      const { chunkKey, remainder } = splitId(id);
+      const bucket = byChunk.get(chunkKey);
+      if (bucket) bucket.push(remainder);
+      else byChunk.set(chunkKey, [remainder]);
+    }
+    const gen = await this.currentGen(seg);
+    const perChunk = await mapWithConcurrency(
+      [...byChunk],
+      this.writeConcurrency,
+      async ([chunkKey, remainders]) => {
+        const cold = await this.coldChunk({ ...seg, chunkKey }, gen) ?? this.codec.empty();
+        let claimed = [];
+        await this.readModifyWrite(seg, chunkKey, (delta) => {
+          const present = effective(cold, delta);
+          claimed = remainders.filter((r) => !present.has(r));
+          for (const r of claimed) applyAdd(delta, r);
+        });
+        return claimed.map((r) => joinId(chunkKey, r));
+      }
+    );
+    return perChunk.flat();
+  }
   async has(seg, id, opts) {
     const { chunkKey, remainder } = splitId(id);
     const ref = { ...seg, chunkKey };
@@ -1394,8 +1443,8 @@ var FOOTER = {
   // u32
   flags: 20,
   // u32
-  roaringSerializationId: 24,
-  // u16
+  payloadCodecId: 24,
+  // u16 — was `roaringSerializationId`; same offset, same width, wider meaning
   elementWidth: 26,
   // u8
   containerCodec: 27,
@@ -1423,7 +1472,10 @@ var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
 var FLAG_ENCRYPTED = 1 << 0;
 var FLAG_LITTLE_ENDIAN = 1 << 2;
 var ELEMENT_WIDTH_32 = 32;
-var ROARING_PORTABLE_ID = 1;
+var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
+var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
+  PAYLOAD_CODEC_ROARING_PORTABLE
+]);
 var CONTAINER_CODEC_NONE = 0;
 var DEFAULT_TAIL_BYTES = 256 * 1024;
 var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
@@ -1520,10 +1572,10 @@ var CrbmReader = class _CrbmReader {
         `.crbm element_width ${elementWidth} not supported (v1 reads 32-bit ids; 64-bit is a future major version)`
       );
     }
-    const serializationId = fview.getUint16(FOOTER.roaringSerializationId, true);
-    if (serializationId !== ROARING_PORTABLE_ID) {
+    const payloadCodecId = fview.getUint16(FOOTER.payloadCodecId, true);
+    if (!KNOWN_PAYLOAD_CODEC_IDS.has(payloadCodecId)) {
       throw new UnsupportedError(
-        `.crbm roaring_serialization_id ${serializationId} not supported (v1 is portable=${ROARING_PORTABLE_ID})`
+        `.crbm payload_codec_id ${payloadCodecId} not supported by this build (known: ${[...KNOWN_PAYLOAD_CODEC_IDS].join(", ")}; ${PAYLOAD_CODEC_ROARING_PORTABLE}=roaring portable). A generation written by a different codec is rejected rather than decoded \u2014 a store uses one codec throughout.`
       );
     }
     const containerCodec = footer[FOOTER.containerCodec];
@@ -1818,8 +1870,8 @@ var CrbmWriter = class {
     view.setUint32(FOOTER.indexCrc32c, crc32c(indexRegion), true);
     view.setUint32(FOOTER.flags, FLAG_LITTLE_ENDIAN | (encrypted ? FLAG_ENCRYPTED : 0), true);
     view.setUint16(
-      FOOTER.roaringSerializationId,
-      this.options.roaringSerializationId ?? ROARING_PORTABLE_ID,
+      FOOTER.payloadCodecId,
+      this.options.payloadCodecId ?? PAYLOAD_CODEC_ROARING_PORTABLE,
       true
     );
     footer[FOOTER.elementWidth] = this.options.elementWidth ?? ELEMENT_WIDTH_32;
@@ -2525,8 +2577,10 @@ async function gcOrphanGenerations(ref, deps, options = {}) {
   const current = record.currentGen;
   const gens = [];
   for await (const key of deps.cold.list(ref)) gens.push(key.generation);
-  const below = gens.filter((g) => g < current).sort((a, b) => b - a);
-  const toDelete = below.slice(keep);
+  const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : (
+    // Delete generations below current, except the newest `keep` of them (the grace window).
+    gens.filter((g) => g < current).sort((a, b) => b - a).slice(keep)
+  );
   for (const generation of toDelete) {
     await deps.cold.delete({ namespace: ref.namespace, segment: ref.segment, generation });
   }
@@ -3640,6 +3694,8 @@ var InProcessKeystore = class {
 // src/core/erasure.ts
 var MAX_CAS_ATTEMPTS = 8;
 var MAX_WARM_PASSES = 4;
+var ERASE_CONCURRENCY = 8;
+var MAX_COLD_SWEEPS = 3;
 async function destroySegment(ref, deps, options) {
   if (options.confirmSegment !== ref.segment) {
     throw new ValidationError(
@@ -3698,28 +3754,147 @@ async function eraseNamespace(namespace, deps, options) {
   audit.onEvent({ kind: "namespace.erase", namespace, segmentsShredded });
   return { destroyed };
 }
-async function shredSegment(ref, deps, allowCleartext) {
+async function dropSegment(ref, deps, options) {
+  if (options.confirmSegment !== ref.segment) {
+    throw new ValidationError(
+      `dropSegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental deletion)`
+    );
+  }
+  const base = { segment: ref.segment, namespace: ref.namespace };
+  if (options.dryRun === true) {
+    const record = await deps.registry.get(ref);
+    const wouldDelete = await listGenerations(deps.cold, ref);
+    return {
+      ...base,
+      dropped: false,
+      warmRowsDeleted: 0,
+      generationsDeleted: [],
+      generationsRemaining: [],
+      wouldDelete,
+      wouldDeleteWarmRows: await countWarmRows(deps.warm, ref),
+      wouldCryptoShred: record?.wrappedDeks !== void 0 && record.wrappedDeks.length > 0,
+      cryptoShredded: false,
+      reason: record === null ? wouldDelete.length === 0 ? "absent" : void 0 : record.status === "destroyed" ? "already" : void 0
+    };
+  }
+  let shred = await shredSegment(ref, deps, true, "dropSegment");
+  if (shred.reason === "absent") {
+    const orphans = await listGenerations(deps.cold, ref);
+    if (orphans.length === 0) {
+      return {
+        ...base,
+        dropped: false,
+        warmRowsDeleted: shred.warmRowsDeleted,
+        generationsDeleted: [],
+        generationsRemaining: [],
+        cryptoShredded: false,
+        reason: "absent"
+      };
+    }
+    try {
+      await deps.registry.create(ref, {
+        // The max listed generation, so the row is consistent with what is on disk if anything reads it before
+        // the sweep finishes. It is a tombstone, so no reader resolves through it either way.
+        currentGen: orphans[orphans.length - 1],
+        status: "destroyed"
+      });
+      shred = { ...shred, destroyed: true, reason: void 0 };
+    } catch (err) {
+      if (!isWriteConflictError(err)) throw err;
+      shred = await shredSegment(ref, deps, true, "dropSegment");
+    }
+  }
+  const lateWarmRows = shred.destroyed ? await eraseWarmQuietly(deps.warm, ref) : 0;
+  if (shred.cryptoShredded) {
+    safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
+      kind: "segment.erase",
+      namespace: ref.namespace,
+      segment: ref.segment
+    });
+  }
+  const generationsDeleted = [];
+  for (let pass = 0; pass < MAX_COLD_SWEEPS; pass++) {
+    const present = await listGenerations(deps.cold, ref);
+    if (present.length === 0) break;
+    const outcomes = await mapWithConcurrency(present, ERASE_CONCURRENCY, async (generation) => {
+      try {
+        await deps.cold.delete({ ...ref, generation });
+        return generation;
+      } catch {
+        return null;
+      }
+    });
+    let deletedThisPass = 0;
+    for (const g of outcomes) {
+      if (g !== null) {
+        generationsDeleted.push(g);
+        deletedThisPass += 1;
+      }
+    }
+    if (deletedThisPass === 0) break;
+  }
+  const generationsRemaining = await listGenerations(deps.cold, ref);
+  if (shred.destroyed) {
+    safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
+      kind: "segment.dispose",
+      namespace: ref.namespace,
+      segment: ref.segment,
+      generationsDeleted: generationsDeleted.length
+    });
+  }
+  return {
+    ...base,
+    dropped: shred.destroyed,
+    warmRowsDeleted: shred.warmRowsDeleted + lateWarmRows,
+    generationsDeleted,
+    generationsRemaining,
+    cryptoShredded: shred.cryptoShredded,
+    reason: shred.reason
+  };
+}
+async function listGenerations(cold, ref) {
+  const generations = [];
+  for await (const key of cold.list(ref)) generations.push(key.generation);
+  return generations.sort((a, b) => a - b);
+}
+async function countWarmRows(warm, ref) {
+  let n = 0;
+  for await (const _row of warm.listChunks(ref)) {
+    n += 1;
+  }
+  return n;
+}
+async function eraseWarmQuietly(warm, ref) {
+  try {
+    return await eraseWarm(warm, ref, "dropSegment");
+  } catch (err) {
+    if (!isWriteConflictError(err)) throw err;
+    return 0;
+  }
+}
+async function shredSegment(ref, deps, allowCleartext, op = "destroySegment") {
   const base = { segment: ref.segment, namespace: ref.namespace };
+  let warmRowsDeleted = 0;
   for (let attempt = 0; attempt < MAX_CAS_ATTEMPTS; attempt++) {
     const record = await deps.registry.get(ref);
     if (record === null) {
-      const warmRowsDeleted2 = await eraseWarm(deps.warm, ref);
+      warmRowsDeleted += await eraseWarm(deps.warm, ref, op);
       return {
         ...base,
         destroyed: false,
         cryptoShredded: false,
         reason: "absent",
-        warmRowsDeleted: warmRowsDeleted2
+        warmRowsDeleted
       };
     }
     if (record.status === "destroyed") {
-      const warmRowsDeleted2 = await eraseWarm(deps.warm, ref);
+      warmRowsDeleted += await eraseWarm(deps.warm, ref, op);
       return {
         ...base,
         destroyed: true,
         cryptoShredded: false,
         reason: "already",
-        warmRowsDeleted: warmRowsDeleted2
+        warmRowsDeleted
       };
     }
     const encrypted = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
@@ -3729,10 +3904,10 @@ async function shredSegment(ref, deps, allowCleartext) {
         destroyed: false,
         cryptoShredded: false,
         reason: "cleartext",
-        warmRowsDeleted: 0
+        warmRowsDeleted
       };
     }
-    const warmRowsDeleted = await eraseWarm(deps.warm, ref);
+    warmRowsDeleted += await eraseWarm(deps.warm, ref, op);
     try {
       await deps.registry.compareAndSwap(ref, record.token, {
         status: "destroyed",
@@ -3747,9 +3922,9 @@ async function shredSegment(ref, deps, allowCleartext) {
       if (!isWriteConflictError(err)) throw err;
     }
   }
-  throw new WriteConflictError(`destroySegment: contention shredding "${ref.segment}" \u2014 retry`);
+  throw new WriteConflictError(`${op}: contention shredding "${ref.segment}" \u2014 retry`);
 }
-async function eraseWarm(warm, ref) {
+async function eraseWarm(warm, ref, op = "destroySegment") {
   let deleted = 0;
   let conflicts = 0;
   for (let pass = 0; pass < MAX_WARM_PASSES; pass++) {
@@ -3758,21 +3933,27 @@ async function eraseWarm(warm, ref) {
     for await (const row of warm.listChunks(ref))
       rows.push({ chunkKey: row.chunkKey, token: row.token });
     if (rows.length === 0) break;
-    for (const { chunkKey, token } of rows) {
-      const chunkRef = { namespace: ref.namespace, segment: ref.segment, chunkKey };
-      try {
-        await warm.deleteConditional(chunkRef, token);
-        deleted += 1;
-      } catch (err) {
-        if (!isWriteConflictError(err)) throw err;
-        conflicts += 1;
+    const erased = await mapWithConcurrency(
+      rows,
+      ERASE_CONCURRENCY,
+      async ({ chunkKey, token }) => {
+        const chunkRef = { namespace: ref.namespace, segment: ref.segment, chunkKey };
+        try {
+          await warm.deleteConditional(chunkRef, token);
+          return true;
+        } catch (err) {
+          if (!isWriteConflictError(err)) throw err;
+          return false;
+        }
       }
-    }
+    );
+    for (const ok of erased) if (ok) deleted += 1;
+    conflicts = erased.length - erased.filter(Boolean).length;
     if (conflicts === 0) break;
   }
   if (conflicts > 0) {
     throw new WriteConflictError(
-      `destroySegment: ${conflicts} warm row(s) of "${ref.segment}" were rewritten during every one of ${MAX_WARM_PASSES} erase passes \u2014 the segment was NOT destroyed and its cleartext warm deltas are still readable. Stop writes to this segment and retry.`
+      `${op}: ${conflicts} warm row(s) of "${ref.segment}" were rewritten during every one of ${MAX_WARM_PASSES} erase passes \u2014 the segment was NOT destroyed and its cleartext warm deltas are still readable. Stop writes to this segment and retry.`
     );
   }
   return deleted;
@@ -3781,7 +3962,7 @@ async function eraseWarm(warm, ref) {
 // src/core/consistency.ts
 var DEFAULT_MAX_SCAN_SEGMENTS = 25e4;
 var DEFAULT_CHECK_CONCURRENCY = 8;
-async function listGenerations(cold, ref) {
+async function listGenerations2(cold, ref) {
   const present = /* @__PURE__ */ new Set();
   for await (const key of cold.list(ref)) present.add(key.generation);
   return present;
@@ -3812,7 +3993,7 @@ async function runConsistencyCheck(deps, options = {}) {
     try {
       const live = await deps.registry.get(ref);
       if (!live || live.status === "destroyed") return { kind: "ok" };
-      const present = await listGenerations(deps.cold, ref);
+      const present = await listGenerations2(deps.cold, ref);
       if (present.has(live.currentGen)) return { kind: "ok" };
       return {
         kind: "issue",
@@ -4027,6 +4208,6 @@ var RetryingRegistryDriver = class {
   }
 };
 
-export { AWS_US_EAST_1_ONDEMAND, BoundedLru, BufferReader, BufferSink, CountingMetricsSink, CrbmColdChunkSource, CrbmReader, CrbmWriter, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, InProcessKeystore, LocalFsColdDriver, LocalFsRegistryDriver, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, MemoryWarmDriver, NOOP_AUDIT, NOOP_METRICS, NodeAead, RecordingAuditSink, RetryingColdChunkSource, RetryingColdDriver, RetryingRegistryDriver, RetryingWarmDriver, SegmentEngine, aadFor, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isTransient, joinId, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, withRetry, writeCrbmGeneration };
+export { AWS_US_EAST_1_ONDEMAND, BoundedLru, BufferReader, BufferSink, CountingMetricsSink, CrbmColdChunkSource, CrbmReader, CrbmWriter, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, InProcessKeystore, LocalFsColdDriver, LocalFsRegistryDriver, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, MemoryWarmDriver, NOOP_AUDIT, NOOP_METRICS, NodeAead, RecordingAuditSink, RetryingColdChunkSource, RetryingColdDriver, RetryingRegistryDriver, RetryingWarmDriver, SegmentEngine, aadFor, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, dropSegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isTransient, joinId, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, withRetry, writeCrbmGeneration };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map