@cloudbitmaps/core 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -293,7 +293,7 @@ declare class BoundedLru<K, V> {
293
293
  * "Observability" section.
294
294
  */
295
295
  /** The segment operations that emit an `op` latency event (timed with the injected clock, at the facade). */
296
- type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'has' | 'count' | 'intersectInto' | 'unionInto' | 'andNotInto';
296
+ type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'claimMany' | 'has' | 'count' | 'intersectInto' | 'unionInto' | 'andNotInto';
297
297
  /**
298
298
  * One observability event. A discriminated union on `kind` — new variants can be added over time without
299
299
  * breaking consumers (they switch on the kinds they care about). `namespace` is the segment's namespace
@@ -634,6 +634,31 @@ declare class SegmentEngine {
634
634
  remove(seg: SegmentRef, id: number): Promise<void>;
635
635
  addMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
636
636
  removeMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
637
+ /**
638
+ * **Atomically claim ids: add them, and return only the ones that were not already there.**
639
+ *
640
+ * The durable analogue of Redis `SETBIT` returning the prior bit — the primitive an exactly-once "have I
641
+ * already processed / already sent to this id?" check needs. `has()` then `add()` cannot do this: two workers
642
+ * can both read absent and both proceed.
643
+ *
644
+ * **Bulk on purpose.** The per-id shape is the one this library is worst at, because a Warm write rewrites a
645
+ * whole 64K-id chunk bitmap: claiming 5,000 ids one at a time costs ~5,000 writes and ~23 MB, and claiming
646
+ * them in one call costs 1 write and ~8 KB. So this takes a batch and does one OCC read-modify-write per
647
+ * distinct chunk — Redis's semantics at this library's cost shape, rather than Redis's cost shape.
648
+ *
649
+ * **Atomicity is per id, which is the guarantee that matters here.** Each id lives in exactly one chunk, and a
650
+ * chunk is one OCC row, so for any id exactly one concurrent claimer sees it as new. It is NOT atomic across
651
+ * chunks (neither is `addMany`) — a mid-flight failure can leave some chunks claimed. Re-running is safe:
652
+ * already-claimed ids simply come back as not-new.
653
+ *
654
+ * **The presence test consults Cold, not just the Warm delta.** Checking `adds` alone would report an id as
655
+ * newly claimed when a compaction had already folded it into a Cold generation — silently breaking
656
+ * exactly-once on any segment that has ever been compacted. The check is the full effective set,
657
+ * `(cold ∪ adds) \ removes`.
658
+ *
659
+ * Returns the claimed ids grouped nowhere in particular — order is unspecified.
660
+ */
661
+ claimMany(seg: SegmentRef, ids: Iterable<number>): Promise<number[]>;
637
662
  has(seg: SegmentRef, id: number, opts?: WarmReadOptions): Promise<boolean>;
638
663
  count(seg: SegmentRef): Promise<number>;
639
664
  /** Whether the Cold source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
@@ -1030,6 +1055,28 @@ type AuditEvent = {
1030
1055
  readonly kind: 'segment.erase';
1031
1056
  readonly namespace?: string;
1032
1057
  readonly segment: string;
1058
+ } | {
1059
+ /**
1060
+ * A segment was **disposed of** — tombstoned and its storage reclaimed (Warm rows + Cold generations
1061
+ * deleted) by `dropSegment`, *without* a key shred.
1062
+ *
1063
+ * Deliberately a separate kind from {@link AuditEvent} `segment.erase`, and the distinction is the point.
1064
+ * `segment.erase` attests that bytes are unreadable **everywhere, backups included** — the only claim that
1065
+ * survives WORM. Deleting an object is strictly weaker: a noncurrent version, a cross-region replica or a
1066
+ * PITR snapshot can still hold the cleartext. Emitting one kind for both would make a compliance dashboard
1067
+ * over-attest, so a cleartext drop gets this instead.
1068
+ *
1069
+ * An **encrypted** segment dropped via `dropSegment` emits **both** — `segment.erase` for the key shred and
1070
+ * this for the storage reclamation — because both things genuinely happened.
1071
+ *
1072
+ * `generationsDeleted` is how many Cold generations went. It can be 0 (a segment whose bytes were already
1073
+ * gone), and it does not promise the storage is now fully reclaimed: check `DropResult.generationsRemaining`
1074
+ * for that.
1075
+ */
1076
+ readonly kind: 'segment.dispose';
1077
+ readonly namespace?: string;
1078
+ readonly segment: string;
1079
+ readonly generationsDeleted: number;
1033
1080
  } | {
1034
1081
  /** A namespace erasure was executed. `segmentsShredded` is the count actually crypto-shredded (may be 0). */
1035
1082
  readonly kind: 'namespace.erase';
@@ -1178,6 +1225,16 @@ declare function compactSegment(ref: SegmentRef, deps: CompactionDeps, options:
1178
1225
  * Garbage-collect superseded Cold generations for a segment: everything strictly below `currentGen`, keeping
1179
1226
  * the most recent `keep` of them as a grace window for in-flight readers pinned to a just-superseded
1180
1227
  * generation (**I5**). Generations ≥ `currentGen` are never touched. Returns the generations deleted.
1228
+ *
1229
+ * **Except on a `destroyed` segment, where EVERY generation is garbage** and the grace window is meaningless.
1230
+ * A tombstoned segment resolves no generation at all, so no reader is or can become pinned to one; and nothing
1231
+ * else in the library would ever collect them — the reconcile path that deletes generations above `currentGen`
1232
+ * returns early on a destroyed row, so without this those objects are billed forever.
1233
+ *
1234
+ * That state is reachable in practice: a `dropSegment` whose Cold sweep threw part-way, or a compaction that was
1235
+ * already staging when the tombstone landed and finished writing its object afterwards. `dropSegment` re-sweeps
1236
+ * and reports whatever it could not reclaim in `generationsRemaining`, but a drop that was never re-run leaves a
1237
+ * residual, and this is what eventually collects it under a running daemon.
1181
1238
  */
1182
1239
  declare function gcOrphanGenerations(ref: SegmentRef, deps: Pick<CompactionDeps, 'cold' | 'registry'>, options?: {
1183
1240
  keep?: number;
@@ -2080,6 +2137,140 @@ declare function eraseNamespace(namespace: string, deps: EraseDeps, options: {
2080
2137
  }): Promise<{
2081
2138
  destroyed: DestroyResult[];
2082
2139
  }>;
2140
+ /**
2141
+ * Deps for {@link dropSegment}. Adds `cold`, because unlike a crypto-shred this one deletes the objects.
2142
+ */
2143
+ interface DropDeps extends EraseDeps {
2144
+ readonly cold: IColdDriver;
2145
+ }
2146
+ interface DropResult {
2147
+ readonly segment: string;
2148
+ readonly namespace?: string;
2149
+ /** True iff the segment is now a `destroyed` tombstone (incl. the idempotent already-dropped case). */
2150
+ readonly dropped: boolean;
2151
+ /** Warm rows physically deleted. */
2152
+ readonly warmRowsDeleted: number;
2153
+ /** Cold generations physically deleted, ascending. Empty on a dry run — see {@link DropResult.wouldDelete}. */
2154
+ readonly generationsDeleted: readonly number[];
2155
+ /**
2156
+ * Generations still present in Cold when the sweep gave up, ascending. **Empty is the normal outcome** — a
2157
+ * non-empty value means the storage was NOT fully reclaimed and the drop should be re-run.
2158
+ *
2159
+ * This field exists because its absence was a defect. `dropped: true` with a populated `generationsDeleted`
2160
+ * and no `reason` used to be returned even when a `.crbm` holding the **complete effective set** had just been
2161
+ * left in the bucket by a compaction that was already mid-flight (it stages from data it read before the
2162
+ * tombstone, so its commit fails on the voided lease but its object survives). For a cleartext segment those
2163
+ * bytes are readable — and nothing in the library reclaimed them, because `gcOrphanGenerations` only considers
2164
+ * generations *below* `currentGen` and `checkConsistency` skips destroyed segments. The result was
2165
+ * indistinguishable from a clean drop, so an operator got no signal to re-run. Now they do.
2166
+ */
2167
+ readonly generationsRemaining: readonly number[];
2168
+ /**
2169
+ * On a dry run, the generations that **would** be deleted. `undefined` on a real run.
2170
+ *
2171
+ * This exists because the confirmation guard the erasure calls use — naming the segment twice — protects a
2172
+ * hand-typed literal and nothing else. In the loop this operation is actually *for*
2173
+ * (`for (const day of expired) drop({segment: day}, {confirmSegment: day})`) the same variable appears twice
2174
+ * and the guard is pure ceremony. A dry run is the guard that still works when a machine is calling.
2175
+ *
2176
+ * It is a **snapshot**: a later real run can see a different set.
2177
+ */
2178
+ readonly wouldDelete?: readonly number[];
2179
+ /**
2180
+ * On a dry run, how many Warm rows would be deleted. `undefined` on a real run.
2181
+ *
2182
+ * Previewing only the Cold generations was a gap: Warm rows hold **cleartext** deltas, so for the automated
2183
+ * caller this preview exists for, "how much unencrypted data is about to go" is at least as load-bearing as
2184
+ * the object count.
2185
+ */
2186
+ readonly wouldDeleteWarmRows?: number;
2187
+ /**
2188
+ * On a dry run, whether the real drop would **also** crypto-shred (i.e. the segment is encrypted).
2189
+ * `undefined` on a real run. This is the one fact in the preview that is irreversible *everywhere*, backups
2190
+ * included — a dry run that did not surface it was previewing the recoverable half only.
2191
+ */
2192
+ readonly wouldCryptoShred?: boolean;
2193
+ /**
2194
+ * True iff the segment was encrypted and its wrapped DEK(s) were dropped as part of this call — so its bytes
2195
+ * are unreadable *everywhere*, backups included, not merely deleted from the bucket. Deleting an object does
2196
+ * not reach a noncurrent version, a replica, or a PITR snapshot; discarding the key does.
2197
+ */
2198
+ readonly cryptoShredded: boolean;
2199
+ /**
2200
+ * `'absent'` (no registry row **and** no Cold objects — nothing existed to drop) or `'already'` (already a
2201
+ * tombstone). Absent on a fresh drop.
2202
+ *
2203
+ * Note what `'absent'` no longer means: it used to be returned while the call deleted every Cold generation it
2204
+ * found, with no tombstone written. See {@link dropSegment}'s "the absent case" note.
2205
+ */
2206
+ readonly reason?: string;
2207
+ }
2208
+ /**
2209
+ * **Dispose of a segment: tombstone it, delete its Warm rows, and delete its Cold objects.** Irreversible.
2210
+ *
2211
+ * WHY THIS EXISTS, given {@link destroySegment} already erases. Because `destroySegment` answers a *compliance*
2212
+ * question and this one answers an *operational* question, and they are not the same:
2213
+ *
2214
+ * - `destroySegment` **crypto-shreds** — it discards the key so the bytes are unreadable everywhere including
2215
+ * immutable backups, which is the only erasure that survives WORM. But it **leaves the objects in the
2216
+ * bucket**, still billed, and it *requires* encryption because a cleartext segment has no key to discard.
2217
+ * - `dropSegment` **removes the storage**. It works on a cleartext segment, and on an encrypted one it *also*
2218
+ * drops the DEKs, so it is a strict superset there.
2219
+ *
2220
+ * Before this existed there was no supported way to delete a segment and stop paying for it, and the obvious
2221
+ * workaround — an object-store lifecycle rule on the key prefix — deletes the bytes while the registry still
2222
+ * points at them. That is exactly the `missing-cold-generation` state the DR runbook says not to serve traffic
2223
+ * on, and it surfaces *intermittently*, because a read consults the hot cache before Cold: cached chunks answer
2224
+ * correctly and evicted ones throw. The whole value of this function is that the ordering below cannot be got
2225
+ * wrong by a caller.
2226
+ *
2227
+ * **THE ORDER IS THE CONTRACT — Warm, then registry, then Cold.** Each step is placed against a specific way
2228
+ * the other orders break:
2229
+ *
2230
+ * 1. **Warm first.** A `destroyed` row makes Cold read as empty, but Warm is consulted *separately* and earlier
2231
+ * on the read path — so a tombstone with live Warm deltas still answers `true` for ids in those deltas.
2232
+ * (This is why {@link shredSegment} already orders it this way; the reasoning is inherited, not invented.)
2233
+ * **Necessary, but not sufficient** — see "writes must have stopped" below.
2234
+ * 2. **Registry second.** After the tombstone nothing resolves a generation for this segment, so no reader can
2235
+ * reach for bytes that are about to disappear.
2236
+ * 3. **Cold last, best-effort, and re-swept.** Once the pointer is a tombstone the segment resolves as empty, so
2237
+ * a failure part-way through leaves **orphaned bytes, not a wrong answer.** Orphans cost money and are cleaned
2238
+ * up by re-running; a torn pointer costs correctness and is not self-healing. Given the choice, leak bytes —
2239
+ * but say so: whatever survives the sweep is reported in {@link DropResult.generationsRemaining}.
2240
+ *
2241
+ * **Why step 3 sweeps more than once.** A compaction that was already mid-flight when the tombstone landed will
2242
+ * still finish *staging* a fresh generation, built from data it read beforehand — its commit then fails on the
2243
+ * lease the tombstone voided, but the object it wrote survives, and it holds the complete effective set including
2244
+ * the Warm deltas step 1 just deleted. A single list-then-delete misses it entirely. The re-sweep converges
2245
+ * because the tombstone *is* a hard fence on **starting** a new generation (compaction returns `destroyed`
2246
+ * before it does anything), so only already-in-flight stagings can appear and they are finite.
2247
+ *
2248
+ * **Writes must have stopped.** Nothing here prevents a *new* Warm write after step 1 (the write path is
2249
+ * deliberately uncoupled — see this module's header). For `destroySegment` that is a scope note; here it is
2250
+ * sharper, because a Warm row that lands after the tombstone is **immortal**: compaction refuses to fold or purge
2251
+ * a destroyed segment, so nothing ever reaps it, and a fresh reader will report the dropped segment as non-empty
2252
+ * *forever*. Step 1 therefore runs a second time after the tombstone, which converges for anything in flight at
2253
+ * drop time — but a writer that keeps writing will keep resurrecting the segment. Stop writes first, or re-run.
2254
+ *
2255
+ * **When "reads as empty" starts being true.** Not instantly, for a store that has already read this segment: a
2256
+ * resolved generation is cached and decoded chunks sit in the hot LRU, so an in-flight reader can answer from
2257
+ * cache for a window. A fresh store, or any reader that had not touched the segment, sees empty at once.
2258
+ *
2259
+ * That window is bounded by `coldGenTtlMs` (default 2 s) **only for a reader whose cold source has both a clock
2260
+ * and a registry and a positive TTL** — expiry needs all three. A source built without a clock, or with
2261
+ * `coldGenTtlMs: 0` (documented as "pin forever"), holds its resolved snapshot for its own lifetime; because a
2262
+ * hot-LRU hit never reaches Cold, such a reader can answer `true` for a dropped segment **indefinitely** and must
2263
+ * be restarted. This is the *same* caching that makes the delete-bytes-first ordering fail intermittently rather
2264
+ * than loudly — it cuts both ways.
2265
+ *
2266
+ * `confirmSegment` must equal `ref.segment`, matching `destroySegment`/`eraseNamespace`. For an automated
2267
+ * caller that guard is ceremony — use `dryRun` first, which reports what would go without touching anything.
2268
+ */
2269
+ declare function dropSegment(ref: SegmentRef, deps: DropDeps, options: {
2270
+ confirmSegment: string;
2271
+ dryRun?: boolean;
2272
+ audit?: IAuditSink;
2273
+ }): Promise<DropResult>;
2083
2274
 
2084
2275
  /**
2085
2276
  * Fail-safe cross-tier DR check (gap #11). The registry (`currentGen`, e.g. in
@@ -2324,4 +2515,4 @@ declare class CrbmWriter {
2324
2515
  private buildFooter;
2325
2516
  }
2326
2517
 
2327
- export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
2518
+ export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type DropDeps, type DropResult, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, dropSegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
package/dist/index.d.ts CHANGED
@@ -293,7 +293,7 @@ declare class BoundedLru<K, V> {
293
293
  * "Observability" section.
294
294
  */
295
295
  /** The segment operations that emit an `op` latency event (timed with the injected clock, at the facade). */
296
- type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'has' | 'count' | 'intersectInto' | 'unionInto' | 'andNotInto';
296
+ type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'claimMany' | 'has' | 'count' | 'intersectInto' | 'unionInto' | 'andNotInto';
297
297
  /**
298
298
  * One observability event. A discriminated union on `kind` — new variants can be added over time without
299
299
  * breaking consumers (they switch on the kinds they care about). `namespace` is the segment's namespace
@@ -634,6 +634,31 @@ declare class SegmentEngine {
634
634
  remove(seg: SegmentRef, id: number): Promise<void>;
635
635
  addMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
636
636
  removeMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
637
+ /**
638
+ * **Atomically claim ids: add them, and return only the ones that were not already there.**
639
+ *
640
+ * The durable analogue of Redis `SETBIT` returning the prior bit — the primitive an exactly-once "have I
641
+ * already processed / already sent to this id?" check needs. `has()` then `add()` cannot do this: two workers
642
+ * can both read absent and both proceed.
643
+ *
644
+ * **Bulk on purpose.** The per-id shape is the one this library is worst at, because a Warm write rewrites a
645
+ * whole 64K-id chunk bitmap: claiming 5,000 ids one at a time costs ~5,000 writes and ~23 MB, and claiming
646
+ * them in one call costs 1 write and ~8 KB. So this takes a batch and does one OCC read-modify-write per
647
+ * distinct chunk — Redis's semantics at this library's cost shape, rather than Redis's cost shape.
648
+ *
649
+ * **Atomicity is per id, which is the guarantee that matters here.** Each id lives in exactly one chunk, and a
650
+ * chunk is one OCC row, so for any id exactly one concurrent claimer sees it as new. It is NOT atomic across
651
+ * chunks (neither is `addMany`) — a mid-flight failure can leave some chunks claimed. Re-running is safe:
652
+ * already-claimed ids simply come back as not-new.
653
+ *
654
+ * **The presence test consults Cold, not just the Warm delta.** Checking `adds` alone would report an id as
655
+ * newly claimed when a compaction had already folded it into a Cold generation — silently breaking
656
+ * exactly-once on any segment that has ever been compacted. The check is the full effective set,
657
+ * `(cold ∪ adds) \ removes`.
658
+ *
659
+ * Returns the claimed ids grouped nowhere in particular — order is unspecified.
660
+ */
661
+ claimMany(seg: SegmentRef, ids: Iterable<number>): Promise<number[]>;
637
662
  has(seg: SegmentRef, id: number, opts?: WarmReadOptions): Promise<boolean>;
638
663
  count(seg: SegmentRef): Promise<number>;
639
664
  /** Whether the Cold source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
@@ -1030,6 +1055,28 @@ type AuditEvent = {
1030
1055
  readonly kind: 'segment.erase';
1031
1056
  readonly namespace?: string;
1032
1057
  readonly segment: string;
1058
+ } | {
1059
+ /**
1060
+ * A segment was **disposed of** — tombstoned and its storage reclaimed (Warm rows + Cold generations
1061
+ * deleted) by `dropSegment`, *without* a key shred.
1062
+ *
1063
+ * Deliberately a separate kind from {@link AuditEvent} `segment.erase`, and the distinction is the point.
1064
+ * `segment.erase` attests that bytes are unreadable **everywhere, backups included** — the only claim that
1065
+ * survives WORM. Deleting an object is strictly weaker: a noncurrent version, a cross-region replica or a
1066
+ * PITR snapshot can still hold the cleartext. Emitting one kind for both would make a compliance dashboard
1067
+ * over-attest, so a cleartext drop gets this instead.
1068
+ *
1069
+ * An **encrypted** segment dropped via `dropSegment` emits **both** — `segment.erase` for the key shred and
1070
+ * this for the storage reclamation — because both things genuinely happened.
1071
+ *
1072
+ * `generationsDeleted` is how many Cold generations went. It can be 0 (a segment whose bytes were already
1073
+ * gone), and it does not promise the storage is now fully reclaimed: check `DropResult.generationsRemaining`
1074
+ * for that.
1075
+ */
1076
+ readonly kind: 'segment.dispose';
1077
+ readonly namespace?: string;
1078
+ readonly segment: string;
1079
+ readonly generationsDeleted: number;
1033
1080
  } | {
1034
1081
  /** A namespace erasure was executed. `segmentsShredded` is the count actually crypto-shredded (may be 0). */
1035
1082
  readonly kind: 'namespace.erase';
@@ -1178,6 +1225,16 @@ declare function compactSegment(ref: SegmentRef, deps: CompactionDeps, options:
1178
1225
  * Garbage-collect superseded Cold generations for a segment: everything strictly below `currentGen`, keeping
1179
1226
  * the most recent `keep` of them as a grace window for in-flight readers pinned to a just-superseded
1180
1227
  * generation (**I5**). Generations ≥ `currentGen` are never touched. Returns the generations deleted.
1228
+ *
1229
+ * **Except on a `destroyed` segment, where EVERY generation is garbage** and the grace window is meaningless.
1230
+ * A tombstoned segment resolves no generation at all, so no reader is or can become pinned to one; and nothing
1231
+ * else in the library would ever collect them — the reconcile path that deletes generations above `currentGen`
1232
+ * returns early on a destroyed row, so without this those objects are billed forever.
1233
+ *
1234
+ * That state is reachable in practice: a `dropSegment` whose Cold sweep threw part-way, or a compaction that was
1235
+ * already staging when the tombstone landed and finished writing its object afterwards. `dropSegment` re-sweeps
1236
+ * and reports whatever it could not reclaim in `generationsRemaining`, but a drop that was never re-run leaves a
1237
+ * residual, and this is what eventually collects it under a running daemon.
1181
1238
  */
1182
1239
  declare function gcOrphanGenerations(ref: SegmentRef, deps: Pick<CompactionDeps, 'cold' | 'registry'>, options?: {
1183
1240
  keep?: number;
@@ -2080,6 +2137,140 @@ declare function eraseNamespace(namespace: string, deps: EraseDeps, options: {
2080
2137
  }): Promise<{
2081
2138
  destroyed: DestroyResult[];
2082
2139
  }>;
2140
+ /**
2141
+ * Deps for {@link dropSegment}. Adds `cold`, because unlike a crypto-shred this one deletes the objects.
2142
+ */
2143
+ interface DropDeps extends EraseDeps {
2144
+ readonly cold: IColdDriver;
2145
+ }
2146
+ interface DropResult {
2147
+ readonly segment: string;
2148
+ readonly namespace?: string;
2149
+ /** True iff the segment is now a `destroyed` tombstone (incl. the idempotent already-dropped case). */
2150
+ readonly dropped: boolean;
2151
+ /** Warm rows physically deleted. */
2152
+ readonly warmRowsDeleted: number;
2153
+ /** Cold generations physically deleted, ascending. Empty on a dry run — see {@link DropResult.wouldDelete}. */
2154
+ readonly generationsDeleted: readonly number[];
2155
+ /**
2156
+ * Generations still present in Cold when the sweep gave up, ascending. **Empty is the normal outcome** — a
2157
+ * non-empty value means the storage was NOT fully reclaimed and the drop should be re-run.
2158
+ *
2159
+ * This field exists because its absence was a defect. `dropped: true` with a populated `generationsDeleted`
2160
+ * and no `reason` used to be returned even when a `.crbm` holding the **complete effective set** had just been
2161
+ * left in the bucket by a compaction that was already mid-flight (it stages from data it read before the
2162
+ * tombstone, so its commit fails on the voided lease but its object survives). For a cleartext segment those
2163
+ * bytes are readable — and nothing in the library reclaimed them, because `gcOrphanGenerations` only considers
2164
+ * generations *below* `currentGen` and `checkConsistency` skips destroyed segments. The result was
2165
+ * indistinguishable from a clean drop, so an operator got no signal to re-run. Now they do.
2166
+ */
2167
+ readonly generationsRemaining: readonly number[];
2168
+ /**
2169
+ * On a dry run, the generations that **would** be deleted. `undefined` on a real run.
2170
+ *
2171
+ * This exists because the confirmation guard the erasure calls use — naming the segment twice — protects a
2172
+ * hand-typed literal and nothing else. In the loop this operation is actually *for*
2173
+ * (`for (const day of expired) drop({segment: day}, {confirmSegment: day})`) the same variable appears twice
2174
+ * and the guard is pure ceremony. A dry run is the guard that still works when a machine is calling.
2175
+ *
2176
+ * It is a **snapshot**: a later real run can see a different set.
2177
+ */
2178
+ readonly wouldDelete?: readonly number[];
2179
+ /**
2180
+ * On a dry run, how many Warm rows would be deleted. `undefined` on a real run.
2181
+ *
2182
+ * Previewing only the Cold generations was a gap: Warm rows hold **cleartext** deltas, so for the automated
2183
+ * caller this preview exists for, "how much unencrypted data is about to go" is at least as load-bearing as
2184
+ * the object count.
2185
+ */
2186
+ readonly wouldDeleteWarmRows?: number;
2187
+ /**
2188
+ * On a dry run, whether the real drop would **also** crypto-shred (i.e. the segment is encrypted).
2189
+ * `undefined` on a real run. This is the one fact in the preview that is irreversible *everywhere*, backups
2190
+ * included — a dry run that did not surface it was previewing the recoverable half only.
2191
+ */
2192
+ readonly wouldCryptoShred?: boolean;
2193
+ /**
2194
+ * True iff the segment was encrypted and its wrapped DEK(s) were dropped as part of this call — so its bytes
2195
+ * are unreadable *everywhere*, backups included, not merely deleted from the bucket. Deleting an object does
2196
+ * not reach a noncurrent version, a replica, or a PITR snapshot; discarding the key does.
2197
+ */
2198
+ readonly cryptoShredded: boolean;
2199
+ /**
2200
+ * `'absent'` (no registry row **and** no Cold objects — nothing existed to drop) or `'already'` (already a
2201
+ * tombstone). Absent on a fresh drop.
2202
+ *
2203
+ * Note what `'absent'` no longer means: it used to be returned while the call deleted every Cold generation it
2204
+ * found, with no tombstone written. See {@link dropSegment}'s "the absent case" note.
2205
+ */
2206
+ readonly reason?: string;
2207
+ }
2208
+ /**
2209
+ * **Dispose of a segment: tombstone it, delete its Warm rows, and delete its Cold objects.** Irreversible.
2210
+ *
2211
+ * WHY THIS EXISTS, given {@link destroySegment} already erases. Because `destroySegment` answers a *compliance*
2212
+ * question and this one answers an *operational* question, and they are not the same:
2213
+ *
2214
+ * - `destroySegment` **crypto-shreds** — it discards the key so the bytes are unreadable everywhere including
2215
+ * immutable backups, which is the only erasure that survives WORM. But it **leaves the objects in the
2216
+ * bucket**, still billed, and it *requires* encryption because a cleartext segment has no key to discard.
2217
+ * - `dropSegment` **removes the storage**. It works on a cleartext segment, and on an encrypted one it *also*
2218
+ * drops the DEKs, so it is a strict superset there.
2219
+ *
2220
+ * Before this existed there was no supported way to delete a segment and stop paying for it, and the obvious
2221
+ * workaround — an object-store lifecycle rule on the key prefix — deletes the bytes while the registry still
2222
+ * points at them. That is exactly the `missing-cold-generation` state the DR runbook says not to serve traffic
2223
+ * on, and it surfaces *intermittently*, because a read consults the hot cache before Cold: cached chunks answer
2224
+ * correctly and evicted ones throw. The whole value of this function is that the ordering below cannot be got
2225
+ * wrong by a caller.
2226
+ *
2227
+ * **THE ORDER IS THE CONTRACT — Warm, then registry, then Cold.** Each step is placed against a specific way
2228
+ * the other orders break:
2229
+ *
2230
+ * 1. **Warm first.** A `destroyed` row makes Cold read as empty, but Warm is consulted *separately* and earlier
2231
+ * on the read path — so a tombstone with live Warm deltas still answers `true` for ids in those deltas.
2232
+ * (This is why {@link shredSegment} already orders it this way; the reasoning is inherited, not invented.)
2233
+ * **Necessary, but not sufficient** — see "writes must have stopped" below.
2234
+ * 2. **Registry second.** After the tombstone nothing resolves a generation for this segment, so no reader can
2235
+ * reach for bytes that are about to disappear.
2236
+ * 3. **Cold last, best-effort, and re-swept.** Once the pointer is a tombstone the segment resolves as empty, so
2237
+ * a failure part-way through leaves **orphaned bytes, not a wrong answer.** Orphans cost money and are cleaned
2238
+ * up by re-running; a torn pointer costs correctness and is not self-healing. Given the choice, leak bytes —
2239
+ * but say so: whatever survives the sweep is reported in {@link DropResult.generationsRemaining}.
2240
+ *
2241
+ * **Why step 3 sweeps more than once.** A compaction that was already mid-flight when the tombstone landed will
2242
+ * still finish *staging* a fresh generation, built from data it read beforehand — its commit then fails on the
2243
+ * lease the tombstone voided, but the object it wrote survives, and it holds the complete effective set including
2244
+ * the Warm deltas step 1 just deleted. A single list-then-delete misses it entirely. The re-sweep converges
2245
+ * because the tombstone *is* a hard fence on **starting** a new generation (compaction returns `destroyed`
2246
+ * before it does anything), so only already-in-flight stagings can appear and they are finite.
2247
+ *
2248
+ * **Writes must have stopped.** Nothing here prevents a *new* Warm write after step 1 (the write path is
2249
+ * deliberately uncoupled — see this module's header). For `destroySegment` that is a scope note; here it is
2250
+ * sharper, because a Warm row that lands after the tombstone is **immortal**: compaction refuses to fold or purge
2251
+ * a destroyed segment, so nothing ever reaps it, and a fresh reader will report the dropped segment as non-empty
2252
+ * *forever*. Step 1 therefore runs a second time after the tombstone, which converges for anything in flight at
2253
+ * drop time — but a writer that keeps writing will keep resurrecting the segment. Stop writes first, or re-run.
2254
+ *
2255
+ * **When "reads as empty" starts being true.** Not instantly, for a store that has already read this segment: a
2256
+ * resolved generation is cached and decoded chunks sit in the hot LRU, so an in-flight reader can answer from
2257
+ * cache for a window. A fresh store, or any reader that had not touched the segment, sees empty at once.
2258
+ *
2259
+ * That window is bounded by `coldGenTtlMs` (default 2 s) **only for a reader whose cold source has both a clock
2260
+ * and a registry and a positive TTL** — expiry needs all three. A source built without a clock, or with
2261
+ * `coldGenTtlMs: 0` (documented as "pin forever"), holds its resolved snapshot for its own lifetime; because a
2262
+ * hot-LRU hit never reaches Cold, such a reader can answer `true` for a dropped segment **indefinitely** and must
2263
+ * be restarted. This is the *same* caching that makes the delete-bytes-first ordering fail intermittently rather
2264
+ * than loudly — it cuts both ways.
2265
+ *
2266
+ * `confirmSegment` must equal `ref.segment`, matching `destroySegment`/`eraseNamespace`. For an automated
2267
+ * caller that guard is ceremony — use `dryRun` first, which reports what would go without touching anything.
2268
+ */
2269
+ declare function dropSegment(ref: SegmentRef, deps: DropDeps, options: {
2270
+ confirmSegment: string;
2271
+ dryRun?: boolean;
2272
+ audit?: IAuditSink;
2273
+ }): Promise<DropResult>;
2083
2274
 
2084
2275
  /**
2085
2276
  * Fail-safe cross-tier DR check (gap #11). The registry (`currentGen`, e.g. in
@@ -2324,4 +2515,4 @@ declare class CrbmWriter {
2324
2515
  private buildFooter;
2325
2516
  }
2326
2517
 
2327
- export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
2518
+ export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type DropDeps, type DropResult, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, dropSegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };