@cloudbitmaps/core 0.7.0 → 0.8.0

package/dist/index.js

@@ -414,6 +414,55 @@ var SegmentEngine = class {
   removeMany(seg, ids) {
     return this.mutateMany(seg, ids, applyRemoveAll);
   }
+  /**
+   * **Atomically claim ids: add them, and return only the ones that were not already there.**
+   *
+   * The durable analogue of Redis `SETBIT` returning the prior bit — the primitive an exactly-once "have I
+   * already processed / already sent to this id?" check needs. `has()` then `add()` cannot do this: two workers
+   * can both read absent and both proceed.
+   *
+   * **Bulk on purpose.** The per-id shape is the one this library is worst at, because a Warm write rewrites a
+   * whole 64K-id chunk bitmap: claiming 5,000 ids one at a time costs ~5,000 writes and ~23 MB, and claiming
+   * them in one call costs 1 write and ~8 KB. So this takes a batch and does one OCC read-modify-write per
+   * distinct chunk — Redis's semantics at this library's cost shape, rather than Redis's cost shape.
+   *
+   * **Atomicity is per id, which is the guarantee that matters here.** Each id lives in exactly one chunk, and a
+   * chunk is one OCC row, so for any id exactly one concurrent claimer sees it as new. It is NOT atomic across
+   * chunks (neither is `addMany`) — a mid-flight failure can leave some chunks claimed. Re-running is safe:
+   * already-claimed ids simply come back as not-new.
+   *
+   * **The presence test consults Cold, not just the Warm delta.** Checking `adds` alone would report an id as
+   * newly claimed when a compaction had already folded it into a Cold generation — silently breaking
+   * exactly-once on any segment that has ever been compacted. The check is the full effective set,
+   * `(cold ∪ adds) \ removes`.
+   *
+   * Returns the claimed ids grouped nowhere in particular — order is unspecified.
+   */
+  async claimMany(seg, ids) {
+    const byChunk = /* @__PURE__ */ new Map();
+    for (const id of ids) {
+      const { chunkKey, remainder } = splitId(id);
+      const bucket = byChunk.get(chunkKey);
+      if (bucket) bucket.push(remainder);
+      else byChunk.set(chunkKey, [remainder]);
+    }
+    const gen = await this.currentGen(seg);
+    const perChunk = await mapWithConcurrency(
+      [...byChunk],
+      this.writeConcurrency,
+      async ([chunkKey, remainders]) => {
+        const cold = await this.coldChunk({ ...seg, chunkKey }, gen) ?? this.codec.empty();
+        let claimed = [];
+        await this.readModifyWrite(seg, chunkKey, (delta) => {
+          const present = effective(cold, delta);
+          claimed = remainders.filter((r) => !present.has(r));
+          for (const r of claimed) applyAdd(delta, r);
+        });
+        return claimed.map((r) => joinId(chunkKey, r));
+      }
+    );
+    return perChunk.flat();
+  }
   async has(seg, id, opts) {
     const { chunkKey, remainder } = splitId(id);
     const ref = { ...seg, chunkKey };
@@ -2528,8 +2577,10 @@ async function gcOrphanGenerations(ref, deps, options = {}) {
   const current = record.currentGen;
   const gens = [];
   for await (const key of deps.cold.list(ref)) gens.push(key.generation);
-  const below = gens.filter((g) => g < current).sort((a, b) => b - a);
-  const toDelete = below.slice(keep);
+  const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : (
+    // Delete generations below current, except the newest `keep` of them (the grace window).
+    gens.filter((g) => g < current).sort((a, b) => b - a).slice(keep)
+  );
   for (const generation of toDelete) {
     await deps.cold.delete({ namespace: ref.namespace, segment: ref.segment, generation });
   }
@@ -3643,6 +3694,8 @@ var InProcessKeystore = class {
 // src/core/erasure.ts
 var MAX_CAS_ATTEMPTS = 8;
 var MAX_WARM_PASSES = 4;
+var ERASE_CONCURRENCY = 8;
+var MAX_COLD_SWEEPS = 3;
 async function destroySegment(ref, deps, options) {
   if (options.confirmSegment !== ref.segment) {
     throw new ValidationError(
@@ -3701,28 +3754,147 @@ async function eraseNamespace(namespace, deps, options) {
   audit.onEvent({ kind: "namespace.erase", namespace, segmentsShredded });
   return { destroyed };
 }
-async function shredSegment(ref, deps, allowCleartext) {
+async function dropSegment(ref, deps, options) {
+  if (options.confirmSegment !== ref.segment) {
+    throw new ValidationError(
+      `dropSegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental deletion)`
+    );
+  }
   const base = { segment: ref.segment, namespace: ref.namespace };
+  if (options.dryRun === true) {
+    const record = await deps.registry.get(ref);
+    const wouldDelete = await listGenerations(deps.cold, ref);
+    return {
+      ...base,
+      dropped: false,
+      warmRowsDeleted: 0,
+      generationsDeleted: [],
+      generationsRemaining: [],
+      wouldDelete,
+      wouldDeleteWarmRows: await countWarmRows(deps.warm, ref),
+      wouldCryptoShred: record?.wrappedDeks !== void 0 && record.wrappedDeks.length > 0,
+      cryptoShredded: false,
+      reason: record === null ? wouldDelete.length === 0 ? "absent" : void 0 : record.status === "destroyed" ? "already" : void 0
+    };
+  }
+  let shred = await shredSegment(ref, deps, true, "dropSegment");
+  if (shred.reason === "absent") {
+    const orphans = await listGenerations(deps.cold, ref);
+    if (orphans.length === 0) {
+      return {
+        ...base,
+        dropped: false,
+        warmRowsDeleted: shred.warmRowsDeleted,
+        generationsDeleted: [],
+        generationsRemaining: [],
+        cryptoShredded: false,
+        reason: "absent"
+      };
+    }
+    try {
+      await deps.registry.create(ref, {
+        // The max listed generation, so the row is consistent with what is on disk if anything reads it before
+        // the sweep finishes. It is a tombstone, so no reader resolves through it either way.
+        currentGen: orphans[orphans.length - 1],
+        status: "destroyed"
+      });
+      shred = { ...shred, destroyed: true, reason: void 0 };
+    } catch (err) {
+      if (!isWriteConflictError(err)) throw err;
+      shred = await shredSegment(ref, deps, true, "dropSegment");
+    }
+  }
+  const lateWarmRows = shred.destroyed ? await eraseWarmQuietly(deps.warm, ref) : 0;
+  if (shred.cryptoShredded) {
+    safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
+      kind: "segment.erase",
+      namespace: ref.namespace,
+      segment: ref.segment
+    });
+  }
+  const generationsDeleted = [];
+  for (let pass = 0; pass < MAX_COLD_SWEEPS; pass++) {
+    const present = await listGenerations(deps.cold, ref);
+    if (present.length === 0) break;
+    const outcomes = await mapWithConcurrency(present, ERASE_CONCURRENCY, async (generation) => {
+      try {
+        await deps.cold.delete({ ...ref, generation });
+        return generation;
+      } catch {
+        return null;
+      }
+    });
+    let deletedThisPass = 0;
+    for (const g of outcomes) {
+      if (g !== null) {
+        generationsDeleted.push(g);
+        deletedThisPass += 1;
+      }
+    }
+    if (deletedThisPass === 0) break;
+  }
+  const generationsRemaining = await listGenerations(deps.cold, ref);
+  if (shred.destroyed) {
+    safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
+      kind: "segment.dispose",
+      namespace: ref.namespace,
+      segment: ref.segment,
+      generationsDeleted: generationsDeleted.length
+    });
+  }
+  return {
+    ...base,
+    dropped: shred.destroyed,
+    warmRowsDeleted: shred.warmRowsDeleted + lateWarmRows,
+    generationsDeleted,
+    generationsRemaining,
+    cryptoShredded: shred.cryptoShredded,
+    reason: shred.reason
+  };
+}
+async function listGenerations(cold, ref) {
+  const generations = [];
+  for await (const key of cold.list(ref)) generations.push(key.generation);
+  return generations.sort((a, b) => a - b);
+}
+async function countWarmRows(warm, ref) {
+  let n = 0;
+  for await (const _row of warm.listChunks(ref)) {
+    n += 1;
+  }
+  return n;
+}
+async function eraseWarmQuietly(warm, ref) {
+  try {
+    return await eraseWarm(warm, ref, "dropSegment");
+  } catch (err) {
+    if (!isWriteConflictError(err)) throw err;
+    return 0;
+  }
+}
+async function shredSegment(ref, deps, allowCleartext, op = "destroySegment") {
+  const base = { segment: ref.segment, namespace: ref.namespace };
+  let warmRowsDeleted = 0;
   for (let attempt = 0; attempt < MAX_CAS_ATTEMPTS; attempt++) {
     const record = await deps.registry.get(ref);
     if (record === null) {
-      const warmRowsDeleted2 = await eraseWarm(deps.warm, ref);
+      warmRowsDeleted += await eraseWarm(deps.warm, ref, op);
       return {
         ...base,
         destroyed: false,
         cryptoShredded: false,
         reason: "absent",
-        warmRowsDeleted: warmRowsDeleted2
+        warmRowsDeleted
       };
     }
     if (record.status === "destroyed") {
-      const warmRowsDeleted2 = await eraseWarm(deps.warm, ref);
+      warmRowsDeleted += await eraseWarm(deps.warm, ref, op);
       return {
         ...base,
         destroyed: true,
         cryptoShredded: false,
         reason: "already",
-        warmRowsDeleted: warmRowsDeleted2
+        warmRowsDeleted
       };
     }
     const encrypted = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
@@ -3732,10 +3904,10 @@ async function shredSegment(ref, deps, allowCleartext) {
         destroyed: false,
         cryptoShredded: false,
         reason: "cleartext",
-        warmRowsDeleted: 0
+        warmRowsDeleted
       };
     }
-    const warmRowsDeleted = await eraseWarm(deps.warm, ref);
+    warmRowsDeleted += await eraseWarm(deps.warm, ref, op);
     try {
       await deps.registry.compareAndSwap(ref, record.token, {
         status: "destroyed",
@@ -3750,9 +3922,9 @@ async function shredSegment(ref, deps, allowCleartext) {
       if (!isWriteConflictError(err)) throw err;
     }
   }
-  throw new WriteConflictError(`destroySegment: contention shredding "${ref.segment}" \u2014 retry`);
+  throw new WriteConflictError(`${op}: contention shredding "${ref.segment}" \u2014 retry`);
 }
-async function eraseWarm(warm, ref) {
+async function eraseWarm(warm, ref, op = "destroySegment") {
   let deleted = 0;
   let conflicts = 0;
   for (let pass = 0; pass < MAX_WARM_PASSES; pass++) {
@@ -3761,21 +3933,27 @@ async function eraseWarm(warm, ref) {
     for await (const row of warm.listChunks(ref))
       rows.push({ chunkKey: row.chunkKey, token: row.token });
     if (rows.length === 0) break;
-    for (const { chunkKey, token } of rows) {
-      const chunkRef = { namespace: ref.namespace, segment: ref.segment, chunkKey };
-      try {
-        await warm.deleteConditional(chunkRef, token);
-        deleted += 1;
-      } catch (err) {
-        if (!isWriteConflictError(err)) throw err;
-        conflicts += 1;
+    const erased = await mapWithConcurrency(
+      rows,
+      ERASE_CONCURRENCY,
+      async ({ chunkKey, token }) => {
+        const chunkRef = { namespace: ref.namespace, segment: ref.segment, chunkKey };
+        try {
+          await warm.deleteConditional(chunkRef, token);
+          return true;
+        } catch (err) {
+          if (!isWriteConflictError(err)) throw err;
+          return false;
+        }
       }
-    }
+    );
+    for (const ok of erased) if (ok) deleted += 1;
+    conflicts = erased.length - erased.filter(Boolean).length;
     if (conflicts === 0) break;
   }
   if (conflicts > 0) {
     throw new WriteConflictError(
-      `destroySegment: ${conflicts} warm row(s) of "${ref.segment}" were rewritten during every one of ${MAX_WARM_PASSES} erase passes \u2014 the segment was NOT destroyed and its cleartext warm deltas are still readable. Stop writes to this segment and retry.`
+      `${op}: ${conflicts} warm row(s) of "${ref.segment}" were rewritten during every one of ${MAX_WARM_PASSES} erase passes \u2014 the segment was NOT destroyed and its cleartext warm deltas are still readable. Stop writes to this segment and retry.`
     );
   }
   return deleted;
@@ -3784,7 +3962,7 @@ async function eraseWarm(warm, ref) {
 // src/core/consistency.ts
 var DEFAULT_MAX_SCAN_SEGMENTS = 25e4;
 var DEFAULT_CHECK_CONCURRENCY = 8;
-async function listGenerations(cold, ref) {
+async function listGenerations2(cold, ref) {
   const present = /* @__PURE__ */ new Set();
   for await (const key of cold.list(ref)) present.add(key.generation);
   return present;
@@ -3815,7 +3993,7 @@ async function runConsistencyCheck(deps, options = {}) {
     try {
       const live = await deps.registry.get(ref);
       if (!live || live.status === "destroyed") return { kind: "ok" };
-      const present = await listGenerations(deps.cold, ref);
+      const present = await listGenerations2(deps.cold, ref);
       if (present.has(live.currentGen)) return { kind: "ok" };
       return {
         kind: "issue",
@@ -4030,6 +4208,6 @@ var RetryingRegistryDriver = class {
   }
 };
 
-export { AWS_US_EAST_1_ONDEMAND, BoundedLru, BufferReader, BufferSink, CountingMetricsSink, CrbmColdChunkSource, CrbmReader, CrbmWriter, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, InProcessKeystore, LocalFsColdDriver, LocalFsRegistryDriver, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, MemoryWarmDriver, NOOP_AUDIT, NOOP_METRICS, NodeAead, RecordingAuditSink, RetryingColdChunkSource, RetryingColdDriver, RetryingRegistryDriver, RetryingWarmDriver, SegmentEngine, aadFor, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isTransient, joinId, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, withRetry, writeCrbmGeneration };
+export { AWS_US_EAST_1_ONDEMAND, BoundedLru, BufferReader, BufferSink, CountingMetricsSink, CrbmColdChunkSource, CrbmReader, CrbmWriter, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, InProcessKeystore, LocalFsColdDriver, LocalFsRegistryDriver, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, MemoryWarmDriver, NOOP_AUDIT, NOOP_METRICS, NodeAead, RecordingAuditSink, RetryingColdChunkSource, RetryingColdDriver, RetryingRegistryDriver, RetryingWarmDriver, SegmentEngine, aadFor, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, dropSegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isTransient, joinId, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, withRetry, writeCrbmGeneration };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map