@cloudbitmaps/roaring 0.6.0 → 0.8.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { bulkLoadCrbmGeneration as bulkLoadCrbmGeneration$1, compactSegment as compactSegment$1, runCompactionCycle as runCompactionCycle$1, runExport as runExport$1, CodecBitmap, CodecInterface, BudgetOption, IColdDriver, ColdChunkSource, IWarmDriver, IRegistryDriver, IKeystore, Clock, Rng, RetryPolicy, IMetricsSink, SegmentEngine, SegmentRef, PricingProfile, Workload, Topology, CostReport, IAuditSink, CompactionOptions, CompactionResult, ConsistencyReport, ExportSink, ExportOptions, ExportManifest, EstimateInput } from '@cloudbitmaps/core';
+import { bulkLoadCrbmGeneration as bulkLoadCrbmGeneration$1, compactSegment as compactSegment$1, runCompactionCycle as runCompactionCycle$1, runExport as runExport$1, CodecBitmap, CodecInterface, BudgetOption, IColdDriver, ColdChunkSource, IWarmDriver, IRegistryDriver, IKeystore, Clock, Rng, RetryPolicy, IMetricsSink, SegmentEngine, SegmentRef, PricingProfile, Workload, Topology, CostReport, IAuditSink, CompactionOptions, CompactionResult, DropResult, ConsistencyReport, ExportSink, ExportOptions, ExportManifest, EstimateInput } from '@cloudbitmaps/core';
 export * from '@cloudbitmaps/core';
 
 /**
@@ -114,8 +114,13 @@ declare const roaringCodec: CodecInterface;
  * In-process lifecycle helpers — `compact`, `eraseSubject`, `subjectReport` — reuse the store's own drivers, so
  * you never re-pass them (they need the store built with a raw cold driver + registry). Out-of-process cold
  * writers (the compaction daemon, the seed/bulk-load CLI, `destroySegment`/`eraseNamespace`) wire their own deps
- * against the same drivers — they run in separate processes.md + the
- * getting-started guide.
+ * against the same drivers — they run in separate processes.
+ *
+ * **You do not need any of them to start.** Construct a store and call `add`/`addMany`/`has`/`remove` — a segment
+ * with no Cold generation at all is fully functional, because a read merges `(cold ∪ warm.adds) \ warm.removes`
+ * and an absent Cold tier just makes that merge trivial. `bulkLoadCrbmGeneration` is an **import** path for data
+ * you already have elsewhere, not an initialization step, and compaction is a **cost** optimization, never a
+ * correctness requirement. See the README and the getting-started guide.
  */
 
 /**
@@ -396,6 +401,48 @@ declare class CloudRoaring {
      * a small daemon-less deployment, or right after a targeted `remove`.
      */
     compact(ref: SegmentRef, options: CompactionOptions): Promise<CompactionResult>;
+    /**
+     * **Dispose of a segment — tombstone it, delete its Warm rows, delete its Cold objects.** Irreversible.
+     *
+     * The operation a rolling window needs: `destroySegment` crypto-shreds (bytes unreadable everywhere including
+     * backups, but still sitting in your bucket and still billed, and it requires encryption), while this one
+     * removes the storage and works on a cleartext segment. On an encrypted segment it does both.
+     *
+     * Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
+     * matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
+     * variable twice.
+     *
+     * ```ts
+     * for (const day of expiredDays) {
+     *   // A colon is NOT legal in a name — the family goes in the namespace, the date in the segment.
+     *   const ref = { namespace: 'active-daily', segment: day };
+     *   await store.dropSegment(ref, { confirmSegment: ref.segment });
+     * }
+     * ```
+     *
+     * **Omitting the namespace addresses a different segment and is a silent no-op** — you get
+     * `{ dropped: false, reason: 'absent' }`, not a throw, so a retention loop with that mistake deletes nothing
+     * forever and quietly. Check `reason` in an automated sweep.
+     *
+     * **Inspect `generationsRemaining`.** Empty is the normal outcome; non-empty means the storage was NOT fully
+     * reclaimed and the drop should be re-run. A compaction that was already in flight when the tombstone landed
+     * still finishes staging one more object, so a single sweep can miss it — this call re-sweeps and then reports
+     * whatever it still could not remove rather than returning a result that looks like a clean drop.
+     *
+     * Reads become empty within `coldGenTtlMs` (default 2 s), not instantly: a store that had already read this
+     * segment may answer from its cached generation + hot chunks until that window lapses. A reader that never
+     * touched it sees empty at once. **That bound needs a clock and `coldGenTtlMs > 0`** — a store built without a
+     * clock, or with `coldGenTtlMs: 0` ("pin forever"), holds its resolved snapshot for its own lifetime and can
+     * keep answering `true` for a dropped segment indefinitely; restart it.
+     *
+     * Needs the store built with a **raw cold driver + a registry** (throws {@link UnsupportedError} otherwise),
+     * because it has to enumerate and delete generations — a pre-built `ColdChunkSource` only reads.
+     */
+    dropSegment(ref: SegmentRef, options: {
+        confirmSegment: string;
+        dryRun?: boolean;
+        audit?: IAuditSink;
+    }): Promise<DropResult>;
     /**
      * **Cross-tier DR consistency check (audit gap #11).** After a restore/failover, verify every registered
      * segment's `currentGen` actually has its `.crbm` present in Cold — catching a **torn restore** where the
@@ -494,6 +541,32 @@ declare class Segment {
     remove(id: number): Promise<void>;
     /** Remove many ids, from a **sync or async** iterable. Not atomic across chunks — see {@link addMany}. */
     removeMany(ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
+    /**
+     * **Claim ids atomically — add them, and get back only the ones that were not already there.**
+     *
+     * The durable analogue of Redis `SETBIT` returning the prior bit, which is what an exactly-once *"have I
+     * already sent to / already processed this id?"* check needs. `has()` then `add()` cannot give you this: two
+     * workers can both read absent and both proceed.
+     *
+     * ```ts
+     * const sent = store.segment(day, { namespace: 'sent-daily' });
+     * const toSend = await sent.claimMany(candidateIds);   // only the ids this worker won
+     * for (const id of toSend) enqueue(id);                // ...so nobody else will send them
+     * ```
+     *
+     * **Pass a batch, not one id.** A Warm write rewrites a whole 64K-id chunk bitmap, so claiming ids one at a
+     * time is the single most expensive way to use this library — measured at ~5,000 writes and ~23 MB for 5,000
+     * ids, against 1 write and ~8 KB for the same ids in one call. This method does one OCC read-modify-write per
+     * distinct chunk, giving you Redis's *semantics* without Redis's per-id *cost shape*.
+     *
+     * **Exactly-once holds per id**, which is the guarantee that matters: each id lives in one chunk, a chunk is
+     * one OCC row, so exactly one concurrent claimer sees any given id as new. Like `addMany` it is **not** atomic
+     * across chunks — a mid-flight failure can leave some chunks claimed — but re-running is safe, because
+     * already-claimed ids simply come back as not-new.
+     *
+     * Order of the returned ids is unspecified.
+     */
+    claimMany(ids: Iterable<number>): Promise<number[]>;
     has(id: number): Promise<boolean>;
     count(): Promise<number>;
     iterate(): AsyncIterable<number>;
@@ -559,6 +632,6 @@ declare class Segment {
 }
 
 /** Package version marker. Kept in sync with package.json at release. */
-declare const VERSION = "0.6.0";
+declare const VERSION = "0.8.0";
 
 export { type BaseCombineOptions, CloudRoaring, type CloudRoaringOptions, type CombineIntoOptions, type CombineOptions, type EraseSubjectResult, SafeBitmap, Segment, type SegmentOptions, type SubjectErasureEntry, type SubjectReport, type SubjectSegmentRef, VERSION, bulkLoadCrbmGeneration, compactSegment, roaringCodec, runCompactionCycle, runExport };

package/dist/index.js

@@ -1,4 +1,4 @@
-import { IntegrityError, bulkLoadCrbmGeneration as bulkLoadCrbmGeneration$1, compactSegment as compactSegment$1, runCompactionCycle as runCompactionCycle$1, runExport as runExport$1, safeMetrics, NOOP_METRICS, BoundedLru, RetryingWarmDriver, RetryingColdChunkSource, ValidationError, resolveBudget, DEFAULT_BUDGET, SegmentEngine, UnsupportedError, validateSegmentRef, resolvePerOpBudget, splitId, collectWithinBudget, mapWithConcurrency, validateCompactionOptions, runConsistencyCheck, estimateCost, groundedReport, CrbmColdChunkSource } from '@cloudbitmaps/core';
+import { IntegrityError, bulkLoadCrbmGeneration as bulkLoadCrbmGeneration$1, compactSegment as compactSegment$1, runCompactionCycle as runCompactionCycle$1, runExport as runExport$1, safeMetrics, NOOP_METRICS, BoundedLru, RetryingWarmDriver, RetryingColdChunkSource, ValidationError, resolveBudget, DEFAULT_BUDGET, SegmentEngine, UnsupportedError, validateSegmentRef, resolvePerOpBudget, splitId, collectWithinBudget, mapWithConcurrency, validateCompactionOptions, gcOrphanGenerations, dropSegment, runConsistencyCheck, estimateCost, groundedReport, CrbmColdChunkSource } from '@cloudbitmaps/core';
 export * from '@cloudbitmaps/core';
 import roaring from 'roaring';
 
@@ -305,7 +305,7 @@ var CloudRoaring = class {
   compactionDeps() {
     if (this.coldDriver === void 0) {
       throw new UnsupportedError(
-        "compact/eraseSubject/checkConsistency need the store built with a raw cold driver (IColdDriver), not a pre-built ColdChunkSource \u2014 or call the compactSegment/runConsistencyCheck free functions with explicit deps"
+        "compact/dropSegment/eraseSubject/checkConsistency need the store built with a raw cold driver (IColdDriver), not a pre-built ColdChunkSource \u2014 or call the compactSegment/runConsistencyCheck free functions with explicit deps"
       );
     }
     if (this.registry === void 0) {
@@ -466,7 +466,56 @@ var CloudRoaring = class {
    */
   async compact(ref, options) {
     validateSegmentRef(ref);
-    return compactSegment$1(ref, this.compactionDeps(), options);
+    const deps = this.compactionDeps();
+    const result = await compactSegment$1(ref, deps, options);
+    if (result.compacted) {
+      await gcOrphanGenerations(ref, { cold: deps.cold, registry: deps.registry }).catch(
+        () => void 0
+      );
+    }
+    return result;
+  }
+  /**
+   * **Dispose of a segment — tombstone it, delete its Warm rows, delete its Cold objects.** Irreversible.
+   *
+   * The operation a rolling window needs: `destroySegment` crypto-shreds (bytes unreadable everywhere including
+   * backups, but still sitting in your bucket and still billed, and it requires encryption), while this one
+   * removes the storage and works on a cleartext segment. On an encrypted segment it does both.
+   *
+   * Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
+   * matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
+   * variable twice.
+   *
+   * ```ts
+   * for (const day of expiredDays) {
+   *   // A colon is NOT legal in a name — the family goes in the namespace, the date in the segment.
+   *   const ref = { namespace: 'active-daily', segment: day };
+   *   await store.dropSegment(ref, { confirmSegment: ref.segment });
+   * }
+   * ```
+   *
+   * **Omitting the namespace addresses a different segment and is a silent no-op** — you get
+   * `{ dropped: false, reason: 'absent' }`, not a throw, so a retention loop with that mistake deletes nothing
+   * forever and quietly. Check `reason` in an automated sweep.
+   *
+   * **Inspect `generationsRemaining`.** Empty is the normal outcome; non-empty means the storage was NOT fully
+   * reclaimed and the drop should be re-run. A compaction that was already in flight when the tombstone landed
+   * still finishes staging one more object, so a single sweep can miss it — this call re-sweeps and then reports
+   * whatever it still could not remove rather than returning a result that looks like a clean drop.
+   *
+   * Reads become empty within `coldGenTtlMs` (default 2 s), not instantly: a store that had already read this
+   * segment may answer from its cached generation + hot chunks until that window lapses. A reader that never
+   * touched it sees empty at once. **That bound needs a clock and `coldGenTtlMs > 0`** — a store built without a
+   * clock, or with `coldGenTtlMs: 0` ("pin forever"), holds its resolved snapshot for its own lifetime and can
+   * keep answering `true` for a dropped segment indefinitely; restart it.
+   *
+   * Needs the store built with a **raw cold driver + a registry** (throws {@link UnsupportedError} otherwise),
+   * because it has to enumerate and delete generations — a pre-built `ColdChunkSource` only reads.
+   */
+  async dropSegment(ref, options) {
+    validateSegmentRef(ref);
+    const deps = this.compactionDeps();
+    return dropSegment(ref, { registry: deps.registry, warm: deps.warm, cold: deps.cold }, options);
   }
   /**
    * **Cross-tier DR consistency check (audit gap #11).** After a restore/failover, verify every registered
@@ -565,6 +614,34 @@ var Segment = class {
   removeMany(ids) {
     return this.timed("removeMany", () => this.engine.removeMany(this.ref, ids));
   }
+  /**
+   * **Claim ids atomically — add them, and get back only the ones that were not already there.**
+   *
+   * The durable analogue of Redis `SETBIT` returning the prior bit, which is what an exactly-once *"have I
+   * already sent to / already processed this id?"* check needs. `has()` then `add()` cannot give you this: two
+   * workers can both read absent and both proceed.
+   *
+   * ```ts
+   * const sent = store.segment(day, { namespace: 'sent-daily' });
+   * const toSend = await sent.claimMany(candidateIds);   // only the ids this worker won
+   * for (const id of toSend) enqueue(id);                // ...so nobody else will send them
+   * ```
+   *
+   * **Pass a batch, not one id.** A Warm write rewrites a whole 64K-id chunk bitmap, so claiming ids one at a
+   * time is the single most expensive way to use this library — measured at ~5,000 writes and ~23 MB for 5,000
+   * ids, against 1 write and ~8 KB for the same ids in one call. This method does one OCC read-modify-write per
+   * distinct chunk, giving you Redis's *semantics* without Redis's per-id *cost shape*.
+   *
+   * **Exactly-once holds per id**, which is the guarantee that matters: each id lives in one chunk, a chunk is
+   * one OCC row, so exactly one concurrent claimer sees any given id as new. Like `addMany` it is **not** atomic
+   * across chunks — a mid-flight failure can leave some chunks claimed — but re-running is safe, because
+   * already-claimed ids simply come back as not-new.
+   *
+   * Order of the returned ids is unspecified.
+   */
+  claimMany(ids) {
+    return this.timed("claimMany", () => this.engine.claimMany(this.ref, ids));
+  }
   has(id) {
     return this.timed("has", () => this.engine.has(this.ref, id));
   }
@@ -681,7 +758,7 @@ var Segment = class {
     });
   }
 };
-var VERSION = "0.6.0";
+var VERSION = "0.8.0";
 
 export { CloudRoaring, SafeBitmap, Segment, VERSION, bulkLoadCrbmGeneration, compactSegment, roaringCodec, runCompactionCycle, runExport };
 //# sourceMappingURL=index.js.map