@cloudbitmaps/core 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/README.md +7 -12
  2. package/dist/{chunk-OSU7W2Y3.js → chunk-2B2QF4FU.js} +205 -756
  3. package/dist/chunk-2B2QF4FU.js.map +7 -0
  4. package/dist/chunk-45ML6YU4.js +641 -0
  5. package/dist/chunk-45ML6YU4.js.map +7 -0
  6. package/dist/chunk-BUDAVGBI.js +73 -0
  7. package/dist/chunk-BUDAVGBI.js.map +7 -0
  8. package/dist/chunk-WPER7R2R.js +121 -0
  9. package/dist/chunk-WPER7R2R.js.map +7 -0
  10. package/dist/core/audit.d.ts +53 -15
  11. package/dist/core/bit-route.d.ts +2 -0
  12. package/dist/core/blob.d.ts +4 -0
  13. package/dist/core/budget.d.ts +8 -3
  14. package/dist/core/check.d.ts +50 -0
  15. package/dist/core/codec.d.ts +13 -3
  16. package/dist/core/combine-many.d.ts +11 -0
  17. package/dist/core/consistency.d.ts +11 -2
  18. package/dist/core/cooperative.d.ts +15 -5
  19. package/dist/core/crbm/reader.d.ts +1 -1
  20. package/dist/core/crbm-storage-source.d.ts +7 -4
  21. package/dist/core/crypto.d.ts +5 -0
  22. package/dist/core/discover.d.ts +5 -3
  23. package/dist/core/engine.d.ts +5 -5
  24. package/dist/core/erase-id.d.ts +36 -13
  25. package/dist/core/erasure.d.ts +61 -27
  26. package/dist/core/errors.d.ts +30 -0
  27. package/dist/core/leases.d.ts +2 -0
  28. package/dist/core/load-input.d.ts +49 -1
  29. package/dist/core/load.d.ts +23 -2
  30. package/dist/core/metrics.d.ts +21 -0
  31. package/dist/core/pinned-storage-source.d.ts +27 -8
  32. package/dist/core/pointer-id.d.ts +9 -0
  33. package/dist/core/ports.d.ts +100 -7
  34. package/dist/core/preflight.d.ts +80 -0
  35. package/dist/core/retention-sweep.d.ts +60 -15
  36. package/dist/core/retention.d.ts +17 -6
  37. package/dist/core/retry.d.ts +1 -2
  38. package/dist/core/rollback.d.ts +22 -12
  39. package/dist/core/validate.d.ts +33 -2
  40. package/dist/driver-kit.d.ts +3 -3
  41. package/dist/driver-kit.js +61 -20
  42. package/dist/driver-kit.js.map +2 -2
  43. package/dist/drivers/_shared/object-registry.d.ts +24 -1
  44. package/dist/drivers/_shared/options.d.ts +15 -7
  45. package/dist/drivers/backends.d.ts +17 -2
  46. package/dist/drivers/crypto.d.ts +17 -2
  47. package/dist/export/index.d.ts +16 -2
  48. package/dist/index.d.ts +10 -9
  49. package/dist/index.js +1799 -1700
  50. package/dist/index.js.map +4 -4
  51. package/dist/node.d.ts +14 -0
  52. package/dist/node.js +1064 -0
  53. package/dist/node.js.map +7 -0
  54. package/package.json +11 -3
  55. package/dist/chunk-OSU7W2Y3.js.map +0 -7
@@ -0,0 +1,50 @@
1
+ /**
2
+ * A value as a message shows it: a number or boolean as itself, a bigint with an `n` (so `5n` is not mistaken for
3
+ * `5`), a string quoted (so `'200'` is not mistaken for `200`), and anything else by its type.
4
+ */
5
+ export declare function shown(value: unknown): string;
6
+ /**
7
+ * `value` if it is a safe integer from `min` to `max` (inclusive; `max` defaults to the largest safe integer), else a
8
+ * `ValidationError` naming `what`: `<what> must be an integer from <min> to <max>; got <value>`.
9
+ */
10
+ export declare function requireInteger(value: unknown, bounds: {
11
+ readonly min: number;
12
+ readonly max?: number;
13
+ readonly what: string;
14
+ readonly note?: string;
15
+ }): number;
16
+ /**
17
+ * Refuse a `requireEncryption` that is present and not exactly a boolean. Every site that acts on the flag compares it
18
+ * with `=== true`, so a truthy string would be taken for `true` by one and for `false` by another.
19
+ */
20
+ export declare function checkRequireEncryption(value: unknown): void;
21
+ /** `value` if it is a safe integer of 1 or more, else a `ValidationError` naming `what`. */
22
+ export declare function requirePositiveInteger(value: unknown, what: string): number;
23
+ /**
24
+ * `value` if it is a finite number within the optional `min` and `max` (inclusive), else a `ValidationError` naming
25
+ * `what`: `<what> must be a finite number from <min> to <max>; got <value>`. `NaN` and the infinities are refused.
26
+ */
27
+ export declare function requireFiniteNumber(value: unknown, bounds: {
28
+ readonly min?: number;
29
+ readonly max?: number;
30
+ readonly what: string;
31
+ readonly note?: string;
32
+ }): number;
33
+ /**
34
+ * Refuse an options bag that is not an object, or that holds a key not in `keys`, naming each such key and the keys it
35
+ * takes. `hint`, when given, ends the message, to point at where a likely mistake belongs (`an S3 client goes in
36
+ * \`client\``). Only the bag's own enumerable string keys are read, so a key inherited through a prototype is not
37
+ * checked.
38
+ */
39
+ export declare function refuseUnknownOptions(name: string, options: unknown, keys: readonly string[], hint?: string): void;
40
+ /**
41
+ * The option `name`'s `value`, refused with `ValidationError` unless it is absent or a boolean: a truthy string or a
42
+ * `1` is not taken for `true`, and `null` is not taken for absent. An absent value comes back `undefined`, for the
43
+ * caller's default.
44
+ */
45
+ export declare function requireBool(name: string, value: unknown): boolean | undefined;
46
+ /**
47
+ * The option `name`'s `value`, refused with `ValidationError` unless it is absent or a string with something in it
48
+ * (one that is empty or only whitespace is refused, as is `null`). An absent value comes back `undefined`.
49
+ */
50
+ export declare function requireNonEmptyString(name: string, value: unknown): string | undefined;
@@ -54,7 +54,7 @@ export interface CodecBitmap {
54
54
  toArray(): number[];
55
55
  /**
56
56
  * The set's values as one ascending `Uint32Array` the caller owns: what a batch read converts a chunk with, in one
57
- * native step instead of one iterator step per value.
57
+ * step instead of one iterator step per value.
58
58
  *
59
59
  * **Optional.** A codec without it is read through its iterator, which is correct and slower.
60
60
  *
@@ -125,7 +125,7 @@ export interface EncodedChunk {
125
125
  }
126
126
  /**
127
127
  * A pluggable bitmap codec — the factory the codec-agnostic engine constructs {@link CodecBitmap}s through.
128
- * Implementations **must** size-cap before handing untrusted bytes to any native decoder and use a safe
128
+ * Implementations **must** size-cap before handing untrusted bytes to a decoder and use a safe
129
129
  * (never a trusting/frozen) deserializer — hard invariant 5.
130
130
  */
131
131
  export interface CodecInterface {
@@ -135,7 +135,7 @@ export interface CodecInterface {
135
135
  fromValues(values: Iterable<number>): CodecBitmap;
136
136
  /**
137
137
  * Size-cap (`bytes.length <= maxBytes`) then portable-deserialize. Throws `IntegrityError` when the input
138
- * exceeds the cap or fails to decode — the native decoder is never handed unbounded or unsafe-format input.
138
+ * exceeds the cap or fails to decode — the decoder is never handed unbounded or unsafe-format input.
139
139
  *
140
140
  * "Fails to decode" includes bytes that are the right length and the wrong shape. A bitmap this returns must
141
141
  * satisfy every invariant its own operations assume (for roaring: containers and values in order, runs
@@ -161,6 +161,16 @@ export interface CodecInterface {
161
161
  * The answer is the whole check: an `owns` that answers `true` for anything lets any object through as a bitmap.
162
162
  */
163
163
  owns?(bitmap: unknown): boolean;
164
+ /**
165
+ * Whether an intersect of three or more operands should meet each chunk's operands smallest-first (by `size`).
166
+ *
167
+ * **Optional, and off by default.** A codec whose combine cost depends on the order, because an early operand that is
168
+ * large and of one container kind makes the running result expensive, sets it; the engine then pays one `size` read per
169
+ * operand per chunk to put the sparsest first. A codec whose combines cost about the same in any order, or whose
170
+ * `size` is expensive to read, leaves it out and keeps the operand order it was given. The answer is the
171
+ * same either way.
172
+ */
173
+ readonly intersectSmallestFirst?: boolean;
164
174
  }
165
175
  /**
166
176
  * Resolve a codec that a **public core entry point** was given, failing fast when it is missing.
@@ -163,6 +163,10 @@ export interface MaterializeManyOutputStats {
163
163
  readonly group: number | null;
164
164
  /** Clock time the group's pass began and the output's publish settled (or it failed), when it ran. */
165
165
  readonly startedAt: number | null;
166
+ /**
167
+ * Clock time (ms) when its publish settled or failed, or `null` when the output never ran. Read from the same clock
168
+ * as `startedAt`.
169
+ */
166
170
  readonly endedAt: number | null;
167
171
  /** Chunks the output holds. */
168
172
  readonly chunks: number;
@@ -178,6 +182,7 @@ export interface MaterializeManyOutputStats {
178
182
  export interface MaterializeManyStats {
179
183
  /** Groups run, the re-runs of outputs a lying index made too big included. */
180
184
  readonly groups: number;
185
+ /** The requests the call made, by class. */
181
186
  readonly requests: {
182
187
  /** Range requests sent for operand chunks (retries of one request count once). */
183
188
  readonly rangeReads: number;
@@ -209,6 +214,7 @@ export interface MaterializeManyStats {
209
214
  readonly put: number;
210
215
  };
211
216
  };
217
+ /** The chunk-read budget: the limit in force, the reads the plan needed and the reads made. */
212
218
  readonly budget: {
213
219
  /** The limit used: `null` when lifted. */
214
220
  readonly maxRequests: number | null;
@@ -217,6 +223,9 @@ export interface MaterializeManyStats {
217
223
  /** Chunk reads made. */
218
224
  readonly used: number;
219
225
  };
226
+ /**
227
+ * The memory the call held: its limit, the most the ledger held at once and what the plan holds for the whole call.
228
+ */
220
229
  readonly memory: {
221
230
  readonly maxBufferedBytes: number;
222
231
  /** The most the ledger held at once. */
@@ -224,6 +233,7 @@ export interface MaterializeManyStats {
224
233
  /** What the plan holds for the whole call, charged to the ledger: operand indexes, and each output's keys and expression. */
225
234
  readonly planBytes: number;
226
235
  };
236
+ /** How many chunk keys the call skipped reading. */
227
237
  readonly chunks: {
228
238
  /** Index keys not read because no output needed them there, summed over groups. */
229
239
  readonly pruned: number;
@@ -238,6 +248,7 @@ export interface MaterializeManyStats {
238
248
  };
239
249
  /** By operand name, an object with no prototype, so any name is a key. */
240
250
  readonly operands: Readonly<Record<string, MaterializeManyOperandStats>>;
251
+ /** One entry per output of the call, in the order the call gave them. */
241
252
  readonly outputs: readonly MaterializeManyOutputStats[];
242
253
  }
243
254
  export interface CombineManyRun<R> {
@@ -7,8 +7,8 @@
7
7
  * the affected segment then throw. This scan
8
8
  * detects it up front (run it at startup after a restore) instead of discovering it on the first read.
9
9
  *
10
- * Read-only; bounded fan-out. `destroyed` (crypto-shredded) segments are skipped — their Storage is intentionally
11
- * gone/unreadable, not a torn restore. A segment whose Storage/registry can't be read this pass is recorded in
10
+ * Read-only; bounded fan-out. `destroyed` segments are skipped: no generation resolves through one, so a missing
11
+ * object under it is not a torn restore. A segment whose Storage/registry can't be read this pass is recorded in
12
12
  * `errored` (never aborts the scan). One segment can add several entries to `inconsistent` (its current object, and with
13
13
  * `generations` one per known generation whose object is not the one recorded) and can also be in `errored`, when a
14
14
  * read failed after those were found; with `generations` a segment costs up to one read per entry of its ledger.
@@ -35,8 +35,11 @@ import type { IKeystore } from './crypto.js';
35
35
  * something a modest operator box survives. Raisable, because a ceiling you cannot lift is a landmine.
36
36
  */
37
37
  export { DEFAULT_MAX_SCAN_SEGMENTS } from './registry-scan.js';
38
+ /** One problem the consistency check found with one segment; `issue` says which. */
38
39
  export interface ConsistencyIssue {
40
+ /** The segment's name. */
39
41
  readonly segment: string;
42
+ /** The segment's namespace, or `undefined` for the default one. */
40
43
  readonly namespace?: string;
41
44
  /**
42
45
  * The registry's `currentGen` for the segment. For `missing-storage-generation` it is the generation whose `.crbm` is
@@ -70,11 +73,17 @@ export interface ConsistencyIssue {
70
73
  }
71
74
  /** A segment that could not be checked this pass (Storage/registry read fault) — not proof of a torn restore. */
72
75
  export interface ConsistencyErrorEntry {
76
+ /** The name of the segment that could not be checked. */
73
77
  readonly segment: string;
78
+ /** That segment's namespace, or `undefined` for the default one. */
74
79
  readonly namespace?: string;
75
80
  /** The read error's message (never the raw error, so the report stays serializable/loggable). */
76
81
  readonly error: string;
77
82
  }
83
+ /**
84
+ * What a consistency check found: how many segments it scanned, the issues, the segments it could not read, and, when
85
+ * asked for, the checks it could not make.
86
+ */
78
87
  export interface ConsistencyReport {
79
88
  /** Registered segments scanned (a `destroyed` segment counts as scanned but is never an issue). */
80
89
  readonly checked: number;
@@ -3,13 +3,16 @@
3
3
  *
4
4
  * Everything this library does on the hot read path is small. Bulk-load is the exception: it is the one place
5
5
  * where a single call legitimately occupies the CPU for hundreds of milliseconds, and Node has exactly one thread
6
- * to occupy. Measured on a 1M-id load of 65,536 chunks onto in-memory storage, on a loaded Apple M3 Pro: **518 ms
7
- * wall, and 519 ms during which the event loop did not turn at all** (medians of 20 fresh-process runs). Wire that to a request handler and every other in-flight request on the instance — health
6
+ * to occupy. Measured on the native codec of 0.23.0 (every figure in this header is the native codec's, to be
7
+ * re-measured on the TypeScript codec), on a 1M-id load of 65,536 chunks onto in-memory storage, on a loaded Apple M3
8
+ * Pro: **518 ms wall, and 519 ms during which the event loop did not turn at all** (medians of 20 fresh-process
9
+ * runs). Wire that to a request handler and every other in-flight request on the instance — health
8
10
  * checks included — waits out the whole load.
9
11
  *
10
- * The fix is not to make the work asynchronous. That was measured too, and it is a 7x *regression*: handing each
11
- * chunk's insert to the threadpool costs ~9 µs of dispatch against ~1.5 µs of actual work once ids are spread
12
- * across ~62,000 chunks (636 ms versus 92 ms). The work is genuinely CPU-bound and genuinely small per chunk;
12
+ * The fix is not to make the work asynchronous. That was measured too, on the native codec's threadpool insert, and
13
+ * it was a 7x *regression*: handing each chunk's insert to the threadpool cost ~9 µs of dispatch against ~1.5 µs of
14
+ * actual work once ids are spread across ~62,000 chunks (636 ms versus 92 ms). The TypeScript codec has no
15
+ * threadpool path. The work is genuinely CPU-bound and genuinely small per chunk;
13
16
  * the only thing wrong with it is that it never stops to let anyone else go. So the fix is to keep the work
14
17
  * synchronous and periodically hand the loop back.
15
18
  *
@@ -51,6 +54,13 @@ export interface Yielder {
51
54
  * (~60 yields across a 1M-id load) while still capping any single blocking stretch in the low tens of ms.
52
55
  */
53
56
  export declare const YIELD_EVERY = 1024;
57
+ /**
58
+ * Ids between a load's ingest-loop yields, sorted or not. 16x coarser than the per-chunk cadence because the per-id
59
+ * work is ~40 ns (a `splitId` and an array push) against ~1.3 µs per chunk — matching cadences would put the yield cost
60
+ * on the wrong side of the work it interrupts. Sized against the *slowest* ingest, an in-memory async generator at
61
+ * ~220 ns/id, so the resulting stretch stays a few ms there and well under 1 ms on a plain array.
62
+ */
63
+ export declare const YIELD_EVERY_IDS: number;
54
64
  /**
55
65
  * Build a periodic yield.
56
66
  *
@@ -179,7 +179,7 @@ export declare class CrbmReader {
179
179
  has(chunkKey: number): boolean;
180
180
  /**
181
181
  * Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
182
- * absent. Throws `IntegrityError` on a CRC mismatch (the bytes must never reach the native
182
+ * absent. Throws `IntegrityError` on a CRC mismatch (the bytes must never reach the codec's
183
183
  * deserializer). The returned bytes may be shared (a small generation's chunks are kept by the reader and handed out
184
184
  * again), so callers must not change them.
185
185
  */
@@ -377,7 +377,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
377
377
  * the one read path with no resilience.
378
378
  *
379
379
  * **`status` is re-checked here**, on every open rather than once at pin time. A pin taken before a
380
- * crypto-shred must not keep unwrapping a DEK the shred destroyed; a destroyed segment resolves no generation
380
+ * crypto-shred must not keep unwrapping a DEK whose wrapping the shred removed; a destroyed segment resolves no
381
+ * generation
381
382
  * for anyone, and a pin's read of one fails. That holds from the pin's next open: a reader already open keeps
382
383
  * the key it unwrapped until it is evicted or invalidated.
383
384
  *
@@ -419,7 +420,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
419
420
  /**
420
421
  * Whether the segment is a registered name at all. With a registry: a row that is not a `destroyed` tombstone,
421
422
  * including one with no generation yet (minted by `setRetention` before the first load), which somebody deliberately
422
- * created. A tombstone holds nothing, so it answers `false`. Without a registry there is no registration to
423
+ * created. No generation resolves through a tombstone, so it answers `false`. Without a registry there is no
424
+ * registration to
423
425
  * consult, so the honest answer is whether the bucket holds any generation of it.
424
426
  *
425
427
  * Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
@@ -569,8 +571,9 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
569
571
  *
570
572
  * A *second* NotFound is pathological (GC outrunning resolution) and propagates rather than fabricating an
571
573
  * absent answer — never return a wrong result. `ifGone` is returned when the segment has no committed
572
- * generation to serve: storage is empty, or the row was dropped or crypto-shredded, in which case reading empty
573
- * is the documented outcome rather than a failure.
574
+ * generation to serve: storage is empty, or the row is destroyed (crypto-shredded, destroyed in the clear or
575
+ * dropped), in which case
576
+ * reading empty is the documented outcome rather than a failure.
574
577
  *
575
578
  * **Bounded to exactly two resolve-and-open round trips.** That is a cost contract, not a detail: the retry
576
579
  * re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
@@ -8,8 +8,11 @@
8
8
  */
9
9
  /** One AEAD result: a fresh per-message nonce, the ciphertext, and the authentication tag. */
10
10
  export interface AeadSealed {
11
+ /** The fresh per-message nonce. */
11
12
  readonly nonce: Uint8Array;
13
+ /** The encrypted bytes. */
12
14
  readonly ciphertext: Uint8Array;
15
+ /** The authentication tag. */
13
16
  readonly tag: Uint8Array;
14
17
  }
15
18
  /**
@@ -36,7 +39,9 @@ export interface Aead {
36
39
  * plaintext key material — safe to store in the registry next to the (encrypted) data.
37
40
  */
38
41
  export interface WrappedDek {
42
+ /** The id of the key-encryption key (KEK) this wrapping is under. */
39
43
  readonly keyId: string;
44
+ /** The wrapped data key, an opaque base64 string. */
40
45
  readonly wrapped: string;
41
46
  }
42
47
  /**
@@ -20,9 +20,11 @@ export interface SegmentInfo extends SegmentRef {
20
20
  */
21
21
  readonly currentGen: number | null;
22
22
  /**
23
- * `destroyed` is a tombstone: the row stays so sweeps can still find it, and the generations are gone — on an
24
- * encrypted segment the DEK wrappings went with them, which is what makes the bytes unrecoverable rather than
25
- * merely deleted.
23
+ * `destroyed` is a tombstone: the row stays so sweeps can still find it, and no generation resolves through it. A
24
+ * `dropSegment` deletes the generations (check its `generationsRemaining`); a crypto-shred leaves them in the bucket
25
+ * and removes the DEK wrappings from the row, which makes every copy of the bytes unreadable once no retained copy of
26
+ * the row still holds them, or every KEK that wrapped them is destroyed. A cleartext segment destroyed with
27
+ * `allowCleartext` has no key to remove: its objects stay in the bucket and stay readable.
26
28
  */
27
29
  readonly status: RegistryRecord['status'];
28
30
  }
@@ -346,11 +346,11 @@ export declare class SegmentEngine {
346
346
  * for the range's edge, never a chunk the cache holds or another operand's. The reads, the budget, the metric and the
347
347
  * errors are the id reads' own.
348
348
  */
349
- intersectChunks(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<CombinedChunk>;
349
+ intersectChunks(segs: readonly SegmentRef[], options?: CombineOptions, label?: string): AsyncGenerator<CombinedChunk>;
350
350
  /** {@link union}, as chunks: see {@link intersectChunks}. */
351
- unionChunks(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<CombinedChunk>;
351
+ unionChunks(segs: readonly SegmentRef[], options?: CombineOptions, label?: string): AsyncGenerator<CombinedChunk>;
352
352
  /** {@link andNot}, as chunks: see {@link intersectChunks}. */
353
- andNotChunks(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>): AsyncGenerator<CombinedChunk>;
353
+ andNotChunks(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>, label?: string): AsyncGenerator<CombinedChunk>;
354
354
  /**
355
355
  * Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
356
356
  * absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
@@ -401,7 +401,7 @@ export declare class SegmentEngine {
401
401
  * The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
402
402
  * Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).
403
403
  *
404
- * The accumulator is a **clone** of the first operand's chunk, and that clone is load-bearing: every other
404
+ * The accumulator is a **clone** of one operand's chunk (the first, or for a codec that asks the one with the fewest values), and that clone is load-bearing: every other
405
405
  * bitmap here is the cached, shared Storage instance and must never be mutated, or the cache is poisoned for
406
406
  * every later reader. The remaining operands and the excludes are only *read* by the in-place ops, so they are
407
407
  * used as-is — one clone per key, not one per operand.
@@ -546,7 +546,7 @@ export declare class SegmentEngine {
546
546
  * the read listed the segment's keys: the read then lists them again, and caches as a read begun after the call.
547
547
  *
548
548
  * The decoded-chunk cache is keyed by generation, which handles a *publish* (the new generation misses) but
549
- * not a *destruction*: a segment that was erased from, dropped, shredded or retired has no newer generation
549
+ * not a *destruction*: a segment that was erased from, dropped, destroyed or retired has no newer generation
550
550
  * whose key would miss, so its decoded chunks stay resident and keep answering. That is why the erasing
551
551
  * process itself could report `erased: true` and then still answer `true` for the same id, out of RAM, with
552
552
  * no storage read to intercept.
@@ -5,9 +5,10 @@
5
5
  * There is no per-id delete on an immutable object, and there is no mutable tier to hold a tombstone, so erasure
6
6
  * is what every other write in this library is: a new generation. The current generation is streamed chunk by
7
7
  * chunk through the ascending writer — every chunk decoded, range-checked and re-encoded, the one holding the id
8
- * with that bit cleared — then published fenced on the generation it streamed, and that generation is collected immediately
9
- * (`keep: 0`), so the bit is **physically gone from the bucket when this returns**. Bounded memory: a window of 32
10
- * chunk reads ahead of the writer, never the whole segment.
8
+ * with that bit cleared — then published fenced on the generation it streamed, and that generation is collected
9
+ * immediately (`keep: 0`), so when it answers `erased: true` the bit is **physically gone from the bucket when this
10
+ * returns** (a noncurrent version, a replica or a backup keeps the old object until its own lifecycle removes it).
11
+ * Bounded memory: a window of 32 chunk reads ahead of the writer, never the whole segment.
11
12
  *
12
13
  * The rewrite is the same generation without one id, so it keeps everything else: the new object carries the source's
13
14
  * metadata as it is, and the row's summary of it, built from what was written, counts one id fewer and holds the same
@@ -21,12 +22,17 @@
21
22
  * leaves generations above the pointer that were once current and can be made current again, so a holder can sit
22
23
  * above the pointer as well as below it, and there can be several. Before it reports `erased: true` the call
23
24
  * therefore lists the bucket once more and reads every generation it has not already seen to be clean; any that
24
- * still holds the id makes it report or throw instead. In the ordinary case that list returns one generation, the
25
- * current one, and costs one `list`.
25
+ * still holds the id makes it report or throw instead. A number can be taken again once its object is deleted, so a
26
+ * generation it saw clean is read again too, but for two: the one it found current and its own rewrite, each skipped
27
+ * only while the row, read after that listing, names it with the footer checksum of the object the call read or wrote.
28
+ * In the ordinary case that list returns one generation, the current one, which the row names: one `list` and one
29
+ * read of the row.
26
30
  *
27
31
  * What it does NOT reach: backups, replicas and noncurrent object versions hold the old object until their own
28
- * lifecycle removes it. For an at-rest guarantee that survives those, use encryption plus crypto-shred
29
- * (`destroySegment`), which is per segment — a per-id shred is infeasible because one DEK covers the whole segment.
32
+ * lifecycle removes it. For an at-rest guarantee that reaches those, use encryption plus crypto-shred
33
+ * (`destroySegment`), which is per segment — a per-id shred is infeasible because one DEK covers the whole segment —
34
+ * and is complete once no retained copy of the segment's registry row holds its wrapped key, or every KEK that
35
+ * wrapped it is destroyed.
30
36
  *
31
37
  * Concurrency. This is a **read-modify-write**, not a plain forward-only publish, and that distinction is where
32
38
  * its correctness lives: the new generation is `from` minus one bit, so it is a valid successor to `from` and to
@@ -36,8 +42,8 @@
36
42
  * incarnation of a name, and a name that is retired and re-created restarts at `0`, so the number alone matched
37
43
  * a different segment entirely. Anything else — a load that published in the meantime, or another
38
44
  * erasure that got there first — is reported as `reason: 'superseded'`, `erased: false`, and the caller re-runs
39
- * against the new generation. A writer that claims the same generation *number* first surfaces as a
40
- * `WriteConflictError` (write-once).
45
+ * against the new generation. So is a writer that claims the same generation *number* first (write-once): the put
46
+ * is refused with a `WriteConflictError`, which is read here as the lost race it is.
41
47
  *
42
48
  * The same answer covers the read half, and for the same reason. An erasure collects with `keep: 0`, which takes
43
49
  * every generation below its new pointer — so a racing erasure can delete the generation this call is streaming,
@@ -60,7 +66,7 @@
60
66
  * its number after the winner's object was already in the bucket. Up there nothing collects it, and it was derived
61
67
  * from a generation the winner has replaced, so when the winner was another erasure it still holds the id that
62
68
  * erasure has just reported gone. That object is deleted before this call returns, once its footer matches the
63
- * rewrite's and a renewal of the row has refused every publish fenced on a row read before it, under the version the
69
+ * rewrite's (a read of it that faults is thrown instead, and the object stays) and a renewal of the row has refused every publish fenced on a row read before it, under the version the
64
70
  * footer read reported, unless {@link DISCARD_ATTEMPTS} renewals lose to other writes (a load, a rollback, a
65
71
  * retention write): then it is kept, and can hold the winner's id, until a later call takes it: any erasure that
66
72
  * rewrites the segment (this one's re-run among them, while its id is still in the current generation) numbers above it
@@ -176,8 +182,8 @@ export interface EraseIdResult {
176
182
  */
177
183
  readonly erased: boolean;
178
184
  /**
179
- * Why the id was not erased, when `erased` is false. `'absent'` (no registry row), `'destroyed'` (a crypto-shred
180
- * tombstone — already unreadable), `'no-generation'` (a row with no generation yet when the call read it, its first
185
+ * Why the id was not erased, when `erased` is false. `'absent'` (no registry row), `'destroyed'` (a tombstone; its
186
+ * objects are searched first), `'no-generation'` (a row with no generation yet when the call read it, its first
181
187
  * read: its bucket can hold first loads' objects, none of them found to hold the id, and `collected` names any this
182
188
  * call deleted unsearched, see below),
183
189
  * `'not-member'` (no
@@ -232,7 +238,7 @@ export interface EraseIdResult {
232
238
  * a missing holder as "someone else got there", never as a failure.
233
239
  *
234
240
  * A call that cannot establish the claim **throws** `WriteConflictError` or the underlying fault rather than
235
- * report `erased: true` over bytes that are still there. Four ways that happens: a Storage `delete` fault; a
241
+ * report `erased: true` over bytes that are still there. Four ways that happens: a Storage `delete` fault the call cannot settle by reading the object back; a
236
242
  * collection pass that could not prove the segment was still the same one (which an ordinary retirement landing
237
243
  * mid-call is enough to cause); the same refusal on the path where the current generation did not hold the id
238
244
  * and nothing was published at all; and a generation still holding the id when the bucket is listed at the end
@@ -242,6 +248,13 @@ export interface EraseIdResult {
242
248
  * rewrite discards its own object, when it gets no answer that reading the row can settle, throws the registry's
243
249
  * `TransientError`, and that object is not deleted.
244
250
  *
251
+ * A delete that gets no answer is settled by reading its key: gone, it counts as deleted and is listed in
252
+ * `collected`. Another object under the number, for a delete conditioned on the version searched, is not counted as
253
+ * deleted: the receipt then rests on the listing of the bucket, and says `erased: true` only when that object does
254
+ * not hold the id. When that read fails too, or the delete lands after it, the call reports the fault, and a re-run
255
+ * finds the object gone and lists nothing for the segment: keep the error with the ledger as the record of the
256
+ * deletion.
257
+ *
245
258
  * A rollback that lands between the call's read of the row and one of its deletes, onto the generation being deleted,
246
259
  * leaves the pointer on an object that is gone. The call finds it at its next read of the row, or by reading the row
247
260
  * once more after its last delete, and **throws `NotFoundError`** rather than report anything: every read of the
@@ -277,3 +290,13 @@ export interface EraseIdResult {
277
290
  export declare function eraseIdFromSegment(ref: SegmentRef, id: number, deps: EraseIdDeps, options?: {
278
291
  audit?: IAuditSink;
279
292
  }): Promise<EraseIdResult>;
293
+ /** What the search for an id in an object needs of the store: the objects, the codec, and the read bounds. */
294
+ export type SearchDeps = Pick<EraseIdDeps, 'storage' | 'codec' | 'maxBitmapBytes' | 'readRetry'>;
295
+ /**
296
+ * The newest generation left under a tombstone that holds `id` in the clear, or `null`: the search
297
+ * {@link eraseIdFromSegment} makes under a tombstone, without its deletes. The caller has read the row as `destroyed`;
298
+ * this reads no row. It lists the bucket once and opens objects newest first, a few at a time, stopping at the first
299
+ * that holds the id; an object sealed under a key is no holder. It writes and deletes nothing. A fault on a read is
300
+ * thrown, after the read retry in `deps.readRetry` (the listing is retried too), never read as "not found".
301
+ */
302
+ export declare function findIdUnderTombstone(ref: SegmentRef, id: number, deps: SearchDeps): Promise<number | null>;
@@ -1,33 +1,51 @@
1
1
  /**
2
2
  * Whole-segment erasure: crypto-shred (`destroySegment`, `eraseNamespace`) and disposal (`dropSegment`).
3
3
  *
4
- * GDPR "right to erasure" on immutable, backed-up storage: you can't delete a `.crbm` object from every
5
- * backup, but you can **delete its key**. `destroySegment` drops the segment's wrapped DEK(s) from the registry
6
- * (a CAS to a `destroyed` tombstone) — the encrypted Storage bytes are then permanently unreadable, everywhere,
7
- * forever. **Immediate + irreversible**, so it's gated behind an explicit confirmation (name the exact
8
- * segment/namespace). A `destroyed` segment reads as empty (its DEK is gone). Only works on an **encrypted**
9
- * segment; a cleartext segment has no key to shred.
4
+ * GDPR "right to erasure" on immutable, backed-up storage: you can't delete a `.crbm` object from every backup, but you
5
+ * can **delete its key**. `destroySegment` drops the segment's wrapped DEK(s) from its current registry row (a CAS to a
6
+ * `destroyed` tombstone). Every copy of the encrypted Storage bytes, backups and WORM copies included, is then
7
+ * unreadable once no retained copy of the row still holds the wrapped key (a noncurrent version, a backup or a
8
+ * point-in-time copy of the row), or once every KEK that wrapped it is destroyed; until then a registry restore to a
9
+ * point before the shred makes the segment readable again. **Immediate**, and it cannot be undone through the library
10
+ * (a registry restore to before it can), so it's gated behind an explicit confirmation (the
11
+ * segment's name; the namespace is not checked, so build the ref's `namespace` from a value you have checked). A `destroyed` segment reads as empty: no generation resolves through a destroyed row. Only an
12
+ * **encrypted** segment is crypto-shredded; a cleartext one has no key to shred, and `allowCleartext` tombstones it
13
+ * with its objects left readable.
10
14
  *
11
- * The tombstone is also the fence every writer respects: a load refuses
12
- * a `destroyed` row, so a load racing an erasure cannot resurrect the segment. A single id's erasure is a different
13
- * operation — `eraseIdFromSegment` rewrites the generation without it.
15
+ * The tombstone is also the fence every writer respects: a load refuses a `destroyed` row, so a load racing an erasure
16
+ * cannot resurrect the segment. A single id's erasure is a different operation — `eraseIdFromSegment` rewrites the
17
+ * generation without it.
14
18
  */
15
19
  import { type IAuditSink } from './audit.js';
16
20
  import type { ChurnDeps } from './leases.js';
17
21
  import type { IStorageDriver, IRegistryDriver, RegistryRecord, SegmentRef } from './ports.js';
22
+ /**
23
+ * What the erasure calls need: the registry whose rows they tombstone, and, optionally, the clock and rng they wait out
24
+ * lease churn with.
25
+ */
18
26
  export interface EraseDeps extends ChurnDeps {
27
+ /** The registry holding the segments' rows. */
19
28
  readonly registry: IRegistryDriver;
20
29
  }
30
+ /**
31
+ * What destroying one segment did: a crypto-shred, a cleartext tombstone (the bytes stay readable), nothing for a
32
+ * segment already destroyed or absent, or, with `destroyed: false` and a `reason`, a segment left as it was, which may
33
+ * still hold data. The result of `destroySegment`, and one entry of what `eraseNamespace` returns.
34
+ */
21
35
  export interface DestroyResult {
36
+ /** The segment's name. */
22
37
  readonly segment: string;
38
+ /** The segment's namespace, or `undefined` for the default one. */
23
39
  readonly namespace?: string;
24
40
  /** True iff the segment is now a `destroyed` tombstone (incl. the idempotent already-destroyed case). */
25
41
  readonly destroyed: boolean;
26
42
  /**
27
43
  * True iff **this call** performed a genuine crypto-shred — an encrypted segment whose wrapped DEK(s) were
28
- * dropped, so its Storage bytes are now permanently unreadable. False for a cleartext tombstone (bytes remain
29
- * readable), for the idempotent already-destroyed case, and for an absent segment. This is what the
30
- * `segment.erase` audit event keys off — a cleartext "erase" is not an irreversible destruction.
44
+ * dropped from its current registry row, so its Storage bytes, in any copy, are unreadable once no retained copy
45
+ * of the row holds the key, or every KEK that wrapped it is destroyed. False for a cleartext tombstone (bytes
46
+ * remain readable), for the idempotent already-destroyed case, and for an absent segment. This is what the
47
+ * `segment.erase` audit event keys off — a cleartext "erase" destroys no key. A shred that throws `TransientError`
48
+ * after its read-back also failed may have landed; its re-run answers `'already'` with `cryptoShredded: false`.
31
49
  */
32
50
  readonly cryptoShredded: boolean;
33
51
  /**
@@ -38,14 +56,20 @@ export interface DestroyResult {
38
56
  *
39
57
  * The last two come with `destroyed: false` and mean **the segment still holds data**. A namespace erase
40
58
  * returns them rather than throwing, so its entries have to be inspected — see {@link eraseNamespace}.
59
+ *
60
+ * A shred that throws `TransientError` after its read-back also failed may have landed; its re-run answers
61
+ * `'already'` with `cryptoShredded: false`.
41
62
  */
42
63
  readonly reason?: string;
43
64
  }
44
65
  /**
45
- * Crypto-shred one segment. **Irreversible.** `confirmSegment` must equal `ref.segment` (a guard against an
46
- * accidental destroy — you must name the exact segment). A non-encrypted segment is rejected unless you opt in
47
- * via `allowCleartext` (there's no key to shred; its Storage bytes stay readable — the tombstone only stops the
48
- * segment resolving).
66
+ * Crypto-shred one segment. It cannot be undone through the library (a registry restore to a point before it can).
67
+ * `confirmSegment` must equal `ref.segment` (a guard against an accidental destroy of another name; it checks the
68
+ * name only, not the namespace, so build the ref's `namespace` from a value you have checked). A non-encrypted segment is rejected unless you opt in via `allowCleartext` (there's no key to shred; its
69
+ * Storage bytes stay readable — the tombstone only stops the segment resolving).
70
+ *
71
+ * The options are `confirmSegment`, `allowCleartext` and `audit`. Any other key (`dryRun`, which this call does not
72
+ * take, included) and an `allowCleartext` that is not a boolean throw `ValidationError` before the registry is read.
49
73
  */
50
74
  export declare function destroySegment(ref: SegmentRef, deps: EraseDeps, options: {
51
75
  confirmSegment: string;
@@ -53,14 +77,16 @@ export declare function destroySegment(ref: SegmentRef, deps: EraseDeps, options
53
77
  audit?: IAuditSink;
54
78
  }): Promise<DestroyResult>;
55
79
  /**
56
- * Crypto-shred every segment in a namespace. **Irreversible.** `confirmNamespace` must equal `namespace`.
57
- * Returns a per-segment result (skips cleartext segments unless `allowCleartext`).
80
+ * Crypto-shred every segment in a namespace. It cannot be undone through the library (a registry restore to a point
81
+ * before it can). `confirmNamespace` must equal `namespace`. Returns a per-segment result (skips cleartext segments
82
+ * unless `allowCleartext`).
58
83
  *
59
84
  * **Bounded.** It lists the whole namespace before it destroys anything, and holds the listing resident, so the
60
85
  * listing is capped at `maxScanSegments` (default 250,000), the ceiling every
61
86
  * other fleet scan keeps. A namespace over it throws `BudgetExceededError` **before any segment is destroyed**,
62
87
  * never part-way through; raise `maxScanSegments` when the namespace really is that large and the memory is there.
63
88
  * A `maxScanSegments` that is not a safe whole number of at least 1 throws `ValidationError`, again before anything is destroyed.
89
+ * So does any option it does not take (`dryRun` included) and an `allowCleartext` that is not a boolean.
64
90
  */
65
91
  export declare function eraseNamespace(namespace: string, deps: EraseDeps, options: {
66
92
  confirmNamespace: string;
@@ -76,8 +102,11 @@ export declare function eraseNamespace(namespace: string, deps: EraseDeps, optio
76
102
  export interface DropDeps extends EraseDeps {
77
103
  readonly storage: IStorageDriver;
78
104
  }
105
+ /** What `dropSegment` did to one segment, or, on a dry run, would do. */
79
106
  export interface DropResult {
107
+ /** The segment's name. */
80
108
  readonly segment: string;
109
+ /** The segment's namespace, or `undefined` for the default one. */
81
110
  readonly namespace?: string;
82
111
  /**
83
112
  * **True iff this segment is now empty as a result of this call — or already was.** This is the field a
@@ -111,14 +140,16 @@ export interface DropResult {
111
140
  readonly wouldDelete?: readonly number[];
112
141
  /**
113
142
  * On a dry run, whether the real drop would **also** crypto-shred (i.e. the segment is encrypted).
114
- * `undefined` on a real run. This is the one fact in the preview that is irreversible *everywhere*, backups
115
- * included — a dry run that did not surface it was previewing the recoverable half only.
143
+ * `undefined` on a real run. This is the one fact in the preview that reaches every copy of the objects, backups
144
+ * and WORM copies included (once no retained copy of the row holds the key, or every KEK that wrapped it is
145
+ * destroyed) — a dry run that did not surface it would preview the recoverable half only.
116
146
  */
117
147
  readonly wouldCryptoShred?: boolean;
118
148
  /**
119
- * True iff the segment was encrypted and its wrapped DEK(s) were dropped as part of this call — so its bytes
120
- * are unreadable *everywhere*, backups included, not merely deleted from the bucket. Deleting an object does
121
- * not reach a noncurrent version, a replica, or a PITR snapshot; discarding the key does.
149
+ * True iff the segment was encrypted and its wrapped DEK(s) were dropped from its current registry row as part of
150
+ * this call — so its bytes are unreadable in every copy, backups included, once no retained copy of the row holds
151
+ * the key or every KEK that wrapped it is destroyed, not merely deleted from the bucket. Deleting an object does
152
+ * not reach a noncurrent version, a replica, or a PITR snapshot; removing the key does, on those terms.
122
153
  */
123
154
  readonly cryptoShredded: boolean;
124
155
  /**
@@ -133,14 +164,17 @@ export interface DropResult {
133
164
  readonly reason?: string;
134
165
  }
135
166
  /**
136
- * **Dispose of a segment: tombstone it, then delete its Storage objects.** Irreversible.
167
+ * **Dispose of a segment: tombstone it, then delete its Storage objects.** It cannot be undone through the library
168
+ * (a restore from the registry's and the bucket's versions or backups can).
137
169
  *
138
170
  * WHY THIS EXISTS, given {@link destroySegment} already erases. Because `destroySegment` answers a *compliance*
139
171
  * question and this one answers an *operational* question, and they are not the same:
140
172
  *
141
- * - `destroySegment` **crypto-shreds** — it discards the key so the bytes are unreadable everywhere including
142
- * immutable backups, which is the only erasure that survives WORM. But it **leaves the objects in the
143
- * bucket**, still billed, and it *requires* encryption because a cleartext segment has no key to discard.
173
+ * - `destroySegment` **crypto-shreds** — it removes the key from the segment's current registry row, so the bytes
174
+ * are unreadable everywhere, immutable backups included, once no retained copy of the row holds the key or every
175
+ * KEK that wrapped it is destroyed: the only erasure that reaches WORM copies of the objects. But it **leaves the
176
+ * objects in the bucket**, still billed, and it *requires* encryption because a cleartext segment has no key to
177
+ * discard.
144
178
  * - `dropSegment` **removes the storage**. It works on a cleartext segment, and on an encrypted one it *also*
145
179
  * drops the DEKs, so it is a strict superset there.
146
180
  *