@cloudbitmaps/core 0.19.1 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +85 -0
  2. package/dist/{chunk-RSLVTRJY.js → chunk-IGLIZBR7.js} +982 -281
  3. package/dist/chunk-IGLIZBR7.js.map +7 -0
  4. package/dist/core/absent-operands.d.ts +8 -0
  5. package/dist/core/audit.d.ts +67 -4
  6. package/dist/core/bit-route.d.ts +2 -1
  7. package/dist/core/blob.d.ts +12 -1
  8. package/dist/core/bulk-load.d.ts +148 -0
  9. package/dist/core/cas-retry.d.ts +27 -0
  10. package/dist/core/check.d.ts +22 -0
  11. package/dist/core/chunk-checks.d.ts +6 -1
  12. package/dist/core/chunk-window.d.ts +11 -4
  13. package/dist/core/combine-expr.d.ts +1 -3
  14. package/dist/core/combine-feed.d.ts +2 -16
  15. package/dist/core/combine-many.d.ts +21 -15
  16. package/dist/core/concurrency.d.ts +8 -0
  17. package/dist/core/consistency.d.ts +76 -18
  18. package/dist/core/crbm/fingerprint.d.ts +23 -0
  19. package/dist/core/crbm/format.d.ts +2 -5
  20. package/dist/core/crbm/frame.d.ts +10 -0
  21. package/dist/core/crbm/plan-reads.d.ts +9 -3
  22. package/dist/core/crbm/reader.d.ts +51 -12
  23. package/dist/core/crbm/varint.d.ts +2 -1
  24. package/dist/core/crbm/writer.d.ts +5 -1
  25. package/dist/core/crbm-open.d.ts +154 -0
  26. package/dist/core/crbm-storage-source.d.ts +243 -490
  27. package/dist/core/crbm-write.d.ts +75 -0
  28. package/dist/core/crypto.d.ts +6 -1
  29. package/dist/core/engine.d.ts +239 -47
  30. package/dist/core/erase-id.d.ts +97 -27
  31. package/dist/core/erasure.d.ts +4 -3
  32. package/dist/core/errors.d.ts +8 -2
  33. package/dist/core/generation-gc.d.ts +131 -16
  34. package/dist/core/generation-list.d.ts +11 -0
  35. package/dist/core/id-window.d.ts +46 -0
  36. package/dist/core/invalidation-scope.d.ts +77 -0
  37. package/dist/core/known-generations-shape.d.ts +13 -0
  38. package/dist/core/known-generations.d.ts +147 -0
  39. package/dist/core/leases.d.ts +9 -1
  40. package/dist/core/ledger-seed.d.ts +17 -0
  41. package/dist/core/load.d.ts +17 -36
  42. package/dist/core/lru.d.ts +2 -15
  43. package/dist/core/metrics.d.ts +9 -7
  44. package/dist/core/order.d.ts +2 -0
  45. package/dist/core/pinned-storage-source.d.ts +7 -7
  46. package/dist/core/pointer-id.d.ts +19 -0
  47. package/dist/core/ports.d.ts +210 -45
  48. package/dist/core/publish.d.ts +207 -0
  49. package/dist/core/reader-defaults.d.ts +14 -3
  50. package/dist/core/registry-scan.d.ts +0 -2
  51. package/dist/core/retention-sweep.d.ts +69 -65
  52. package/dist/core/retention.d.ts +1 -1
  53. package/dist/core/retry.d.ts +12 -19
  54. package/dist/core/rollback.d.ts +62 -32
  55. package/dist/core/summary.d.ts +74 -25
  56. package/dist/core/token.d.ts +13 -8
  57. package/dist/core/validate.d.ts +2 -0
  58. package/dist/driver-kit.d.ts +6 -1
  59. package/dist/driver-kit.js +90 -10
  60. package/dist/driver-kit.js.map +3 -3
  61. package/dist/drivers/_shared/bytes.d.ts +6 -0
  62. package/dist/drivers/_shared/faults.d.ts +15 -0
  63. package/dist/drivers/_shared/object-keys.d.ts +18 -0
  64. package/dist/drivers/_shared/object-registry-keys.d.ts +0 -2
  65. package/dist/drivers/_shared/object-registry.d.ts +3 -1
  66. package/dist/drivers/_shared/options.d.ts +19 -0
  67. package/dist/drivers/_shared/registry.d.ts +31 -19
  68. package/dist/drivers/localfs/exact-case.d.ts +0 -3
  69. package/dist/drivers/localfs/fs-util.d.ts +22 -0
  70. package/dist/drivers/localfs/registry.d.ts +10 -4
  71. package/dist/drivers/localfs/storage.d.ts +16 -8
  72. package/dist/drivers/memory.d.ts +14 -9
  73. package/dist/drivers/retry/retrying-chunk-source.d.ts +1 -5
  74. package/dist/export/index.d.ts +16 -0
  75. package/dist/index.d.ts +2 -5
  76. package/dist/index.js +4303 -2938
  77. package/dist/index.js.map +4 -4
  78. package/package.json +1 -1
  79. package/dist/chunk-RSLVTRJY.js.map +0 -7
  80. package/dist/core/cost.d.ts +0 -351
  81. package/dist/core/kept-generations.d.ts +0 -40
@@ -1,14 +1,7 @@
1
- import { type IAuditSink } from './audit.js';
2
- import { ValidationError } from './errors.js';
3
- import type { Yielder } from './cooperative.js';
4
- import type { Clock, Rng } from './determinism.js';
5
- import type { PublishedKept } from './generation-gc.js';
6
- import type { Aead, CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
7
- import type { ChunkRef, ChunkRead, ReadChunksOptions, StorageChunkSource, GenerationMetadata, GenerationSummary, GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, RegistrySummary, SegmentRef, SegmentSize, Token } from './ports.js';
8
- import { CrbmReader } from './crbm/reader.js';
1
+ import type { Clock } from './determinism.js';
2
+ import type { IKeystore } from './crypto.js';
3
+ import type { ChunkRef, ChunkRead, ReadChunksOptions, StorageChunkSource, GenerationSummary, IStorageDriver, IRegistryDriver, RegistryRecord, SegmentRef, SegmentSize, Token } from './ports.js';
9
4
  import type { CrbmReaderOptions } from './crbm/reader.js';
10
- import type { CodecBitmap, CodecInterface } from './codec.js';
11
- import { ChunkLoadInput, DecodedLoadInput } from './load-input.js';
12
5
  export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
13
6
  /**
14
7
  * Optional {@link IRegistryDriver}. When provided, the current generation is resolved via the registry's
@@ -45,7 +38,7 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
45
38
  * How long (ms) a resolved `currentGen` is trusted before the next read re-resolves it (default 2000) — the
46
39
  * bound on read staleness after a load publishes: a reader may serve the prior generation for up to this long,
47
40
  * then converges. Lazy (checked on read — no timer); ≤ one cheap registry read per segment per window, and a
48
- * new {@link CrbmReader} is opened only when the generation actually changed. `0` turns the refresh off, as a
41
+ * new {@link CrbmReader} is opened only when the row's pointer was renewed. `0` turns the refresh off, as a
49
42
  * missing clock or registry does: {@link clock} says what that changes, and what it does not.
50
43
  */
51
44
  readonly currentGenTtlMs?: number;
@@ -53,7 +46,9 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
53
46
  * Hard ceiling on how many segments' readers (each holding a fully-parsed `.crbm` index) are cached at once
54
47
  * (default 1024) — a memory bound for a long-running server. Past it, the least-recently-used
55
48
  * segment's reader is evicted; the next read of an evicted segment re-opens it (one cheap tail GET, since
56
- * generations are immutable). Raise it for a big cache working set of small segments.
49
+ * generations are immutable). Raise it for a big cache working set of small segments, and raise {@link maxOpenIndexBytes} with it: each open reader counts about 3.75 KiB there, so past about 17,000 tiny readers the byte ceiling binds first. A source with a timed refresh
50
+ * also keeps up to 8 times this many segments' resolutions apart from their readers, for the TTL: the re-open then
51
+ * reads no row, and comes only when a read needs the reader, for the index or for a chunk.
57
52
  */
58
53
  readonly maxOpenSegments?: number;
59
54
  /**
@@ -62,31 +57,33 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
62
57
  * **second half of that memory bound**. `maxOpenSegments` alone bounds by *count*, but a wide/dense segment's
63
58
  * parsed index can reach about 1.3 MB, so 1024 wide indices could pin over a GB and blow a small heap (e.g. a 128 MB
64
59
  * Lambda) while the count is nominally "in bounds". This caps the summed {@link CrbmReader.retainedBytes} (the
65
- * parsed index, any metadata a generation carries, and any kept chunk bytes) across cached readers; the least-recently-used reader is evicted once the total would exceed it — whichever
60
+ * parsed index, any metadata a generation carries, and any kept chunk bytes) plus a fixed 3.75 KiB for the objects each open reader
61
+ * holds around them, across cached readers, so that many tiny segments are bounded too; the least-recently-used reader is evicted once the total would exceed it — whichever
66
62
  * of the count/byte bounds binds first. A reader whose open read the whole object keeps a copy of its chunk region when
67
63
  * that region is at most this ceiling divided by {@link maxOpenSegments} (64 KiB by default) — its fair share — so a small generation's
68
64
  * chunks are served from memory with no further request, and a bigger one is read by range. Only a source with a timed
69
65
  * pointer refresh ({@link currentGenTtlMs} above 0, a registry and a clock) keeps chunk bytes. Lower it for memory-tight deployments with wide segments; a single
70
- * segment whose index alone exceeds the budget is still cached (it can't be shrunk) but nothing else alongside.
66
+ * segment whose weight alone exceeds the budget is still cached (it can't be shrunk) but nothing else alongside. A
67
+ * sixteenth of it bounds the resolutions a source with a timed refresh keeps besides (4 MiB by default).
71
68
  */
72
69
  readonly maxOpenIndexBytes?: number;
73
70
  }
74
71
  /**
75
72
  * What a pin holds of the object it pinned: the version its cache entries are keyed by, and the object's
76
- * fingerprint, which a pinned read checks each time it opens the generation. `pinGeneration` always records one.
77
- * A pin built by hand without one is not checked: it reads whatever object is under its generation's key.
73
+ * fingerprint, which a pinned read checks each time it opens the generation. `pinGeneration` always records both,
74
+ * and a pinned read given one without both as strings throws a `ValidationError`.
78
75
  */
79
76
  export interface PinnedObject {
80
77
  readonly version: string;
81
- readonly fingerprint?: string;
78
+ readonly fingerprint: string;
82
79
  }
83
80
  /**
84
- * What a live version's resolution named, without the object's fingerprint: `<generation>:<row token>`, or
81
+ * What a live version's resolution named, without the object's fingerprint: `<generation>:<pointerId>`, or
85
82
  * `<generation>` with no registry. For a caller that compares a live version, from
86
- * {@link CrbmStorageChunkSource.currentVersion} or a chunk of its `getChunks`, with a row it read, which names no
87
- * object. A live version always ends with the suffix, which holds no `#`, so the match starts at its last `#` and a row
88
- * token holding `#` of its own is kept whole. Known limit: a string with no suffix, such as a pin's version or one
89
- * from another source, loses its end when its row token itself ends in `#<base 36>.<base 36>`; no caller passes one.
83
+ * {@link CrbmStorageChunkSource.currentVersion} or a chunk of its `getChunks`, with a row it read. A live version always
84
+ * ends with the suffix, which holds no `#`, so the match starts at its last `#` and a `pointerId` holding `#` of its own
85
+ * is kept whole. Known limit: a string with no suffix, such as a pin's version or one from another source, loses its end
86
+ * when its `pointerId` itself ends in `#<base 36>.<base 36>`; no caller passes one.
90
87
  */
91
88
  export declare function rowVersionOf(version: string): string;
92
89
  /** What a pin that leases its generation does between reading the row and opening the object. */
@@ -96,6 +93,15 @@ export interface PinLease {
96
93
  readonly token: Token;
97
94
  } | 'moved'>;
98
95
  }
96
+ /**
97
+ * The fixed heap an open reader's snapshot holds beyond {@link CrbmReader.retainedBytes}: the snapshot, the reader, its
98
+ * five index arrays as objects, and the closures and promises that tie them. It is added to a reader's weight in the
99
+ * reader cache, so that the byte ceiling bounds many tiny segments as it does a few wide ones. Measured as the heap that
100
+ * stayed (full collections until stable) per segment after a source opened 2,000 one-chunk segments, minus the 20 bytes
101
+ * of index the weight already counts: about 3.7 KiB on Node 24, rounded up to 3.75 KiB.
102
+ * `tests/core/reader-cache-heap.test.ts` holds the counted weight to that heap.
103
+ */
104
+ export declare const OPEN_READER_BYTES = 3840;
99
105
  export declare class CrbmStorageChunkSource implements StorageChunkSource {
100
106
  private readonly driver;
101
107
  /**
@@ -112,6 +118,21 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
112
118
  * steady-state memory bound; re-opening an evicted segment is one cheap tail GET.
113
119
  */
114
120
  private readonly snapshots;
121
+ /**
122
+ * Each segment's resolution, kept apart from its reader, by segment: what a read of its row found, for the TTL from
123
+ * the instant that read was sent, whether or not the reader cache still holds the segment's reader. So letting a reader
124
+ * go moves nothing: the next read builds a snapshot on the resolution, reads no row, and opens the object when it needs
125
+ * the index or a chunk the chunk cache does not hold (a version, a count and a stat from the row's summary need
126
+ * neither). It holds the resolved fields only ({@link Resolved}): the wrapped keys, never an unwrapped one, so a
127
+ * snapshot built on it unwraps the key again through the keystore. It never holds a resolution that found no
128
+ * generation: the next read of such a segment reads the row.
129
+ *
130
+ * Only a source with a timed refresh has one: without it, letting a reader go is one of the few things that ever moves
131
+ * a read on, and keeping the resolution would keep it forever. **Bounded** by count and by bytes, both derived from
132
+ * the reader cache's: {@link RESOLUTIONS_PER_OPEN_SEGMENT} times its count bound, and its byte bound over
133
+ * {@link RESOLUTION_BYTES_DIVISOR}. An entry heavier than the byte bound is kept alone, as the reader cache keeps one.
134
+ */
135
+ private readonly resolutions;
115
136
  /**
116
137
  * Pins ({@link heldKey}) whose object has been found replaced: another object is under its generation's key. Kept,
117
138
  * and bounded as the readers are, so that a pin's later chunk reads and any reopen fail at once rather than pay for
@@ -133,7 +154,12 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
133
154
  * counts every segment's invalidations: a read that waited through another segment's only costs a later read the
134
155
  * check again.
135
156
  */
136
- private invalidations;
157
+ private get invalidations();
158
+ /**
159
+ * Which segments {@link invalidate} has been called for, the latest invalidation of up to 1,024 of them: a pinned
160
+ * stream asks it whether its own segment was invalidated since it last looked.
161
+ */
162
+ private readonly invalidated;
137
163
  /**
138
164
  * Generations whose row summary this process found to disagree with their object, by segment and generation. A
139
165
  * summary here is not used again: a count of that generation reads the object. Bounded as the readers are, so a
@@ -141,6 +167,13 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
141
167
  * the disagreement again.
142
168
  */
143
169
  private readonly distrusted;
170
+ /**
171
+ * The readers held to the row's summary: by a live open ({@link openLive}), or by a live pin that shared one. A pin
172
+ * that finds one memoised under its version shares it as it is: the version names the row's `pointerId`, and every
173
+ * write of the summary renews it, so the summary it was held to is the one the row has now. Any other memoised reader
174
+ * (a `pinAt`'s, held to the fingerprint its caller named) is held to the row's summary before a live pin shares it.
175
+ */
176
+ private readonly rowChecked;
144
177
  private readonly registry;
145
178
  private readonly keystore;
146
179
  private readonly requireEncryption;
@@ -151,8 +184,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
151
184
  private readonly currentGenTtlMs;
152
185
  /**
153
186
  * Whether the segment's pointer is re-read on a timer: it needs a clock (the TTL), a registry (the cheap row read; with
154
- * none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes, the TTL lapse
155
- * and the refresh interval the cost report prices, so it is decided once.
187
+ * none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes and the TTL
188
+ * lapse, so it is decided once.
156
189
  */
157
190
  private readonly timedRefresh;
158
191
  constructor(driver: IStorageDriver, options?: CrbmStorageChunkSourceOptions);
@@ -164,63 +197,90 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
164
197
  private readonly keys;
165
198
  private keyOf;
166
199
  /**
167
- * The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window. The snapshot holds the resolved
168
- * target, and its reader once a read has asked for one.
200
+ * The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window: the snapshot the reader cache holds,
201
+ * while its resolution is inside the TTL. With a timed refresh, a segment whose reader the cache let go is served a
202
+ * snapshot built on its kept resolution, while that is inside the TTL: no row is read, and the reader opens only when a
203
+ * read needs one. Otherwise the pointer is read now. The snapshot holds the resolved target, and its reader once a read
204
+ * has asked for one.
169
205
  */
170
206
  private liveSnapshot;
171
- /** The current resolved reader for a segment, refreshing on the TTL. */
172
- private resolvedReader;
173
207
  /**
174
- * A refresh failed transiently and the prior reader keeps serving: make this snapshot lapse after
175
- * {@link REFRESH_RETRY_MS} (never longer than the TTL) rather than after a whole TTL, so the next read after
176
- * that asks the registry again. It costs no read of its own; it only changes when the next refresh happens.
208
+ * Read the segment's pointer now. The read is sent before this returns, and the TTL counts from this instant: a read
209
+ * that answers late does not lengthen the life of what it found. The row read is always fresh, so what a snapshot on
210
+ * it says of the generation (its summary included) is what the row says now, never `prior`'s.
211
+ *
212
+ * A read that fails with a transient fault answers what `prior` found, when it found a generation, and is asked again
213
+ * soon ({@link retrySoon}): `prior` is the resolution of the snapshot the reader cache still holds, so an outage of the
214
+ * registry keeps that snapshot's generation, and its reader and unwrapped key where it has them; one resolved without
215
+ * an open, for a count or a generation lookup, unwraps the key from `prior`'s wrapped keys when a read first needs
216
+ * it. The outage ends shortly after the registry answers.
217
+ * That answer is the snapshot's alone ({@link Resolution.rodeOut}): it is not kept as the segment's resolution, and a
218
+ * snapshot built on it after the reader cache let that one go fails with the fault. With no `prior` the fault
219
+ * fails the read, as a cold resolve's does. If `prior` found none or failed, the pointer is read once more rather than
220
+ * a dead resolution re-armed (else the segment reads empty for a whole TTL window). Anything else — an access denial, a corrupt row, a
221
+ * registry that answers NotFound — fails the read exactly as a cold resolve of the segment would, and the resolution
222
+ * and the snapshot on it are forgotten, so a reader and the key it unwrapped do not outlive a refresh that could not
223
+ * be trusted.
224
+ */
225
+ private resolveNow;
226
+ /**
227
+ * A refresh failed transiently and the prior resolution keeps serving: make it lapse after {@link REFRESH_RETRY_MS}
228
+ * (never longer than the TTL) rather than after a whole TTL, so the next read after that asks the registry again. It
229
+ * costs no read of its own; it only changes when the next refresh happens.
177
230
  */
178
231
  private retrySoon;
179
232
  /**
180
- * TTL elapsed: cheaply re-resolve the pointer. The row read is always fresh, so what the new snapshot says of the
181
- * generation (its summary included) is what the row says now, never the prior snapshot's. The reader is reopened
182
- * only if the generation or the row changed, and only when a read asks for it.
233
+ * Keep `resolution` as the segment's, from the instant its read was sent, so the reads that come while it is in flight
234
+ * share it, and a read after the reader cache let the segment go builds on it. Its weight is reported once it lands,
235
+ * and one that finds no generation, or fails, is forgotten. Both are identity-guarded, as the reader cache's are, and
236
+ * nothing is installed when the read lands: an `invalidate` while it is in flight, or a newer resolution, is never
237
+ * undone by its answer.
183
238
  */
184
- private refreshedTarget;
239
+ private remember;
185
240
  /**
186
- * Memoize a snapshot as the segment's at `now()`, forgetting it later if it resolves to null / throws
187
- * (identity-guarded, so a stale cleanup never clobbers a fresher snapshot). Installing the (pending) snapshot
188
- * **synchronously** is what lets concurrent callers coalesce onto one in-flight resolve.
241
+ * A live snapshot on `resolution`. `before` is the snapshot it replaces, while the reader cache still holds one: its
242
+ * reader is kept if the resolution finds the same generation, `pointerId` and object ({@link Snapshot.reuse}), and its
243
+ * unwrapped key if the same wrapped keys. A snapshot built after the reader cache let the segment go has neither, so
244
+ * its open asks the keystore again: a key is held only while a reader of the segment is.
189
245
  */
190
- private install;
191
- private now;
246
+ private snapshotOn;
247
+ /** What the snapshot on `resolution` that replaces `before` reads: see {@link snapshotOn}. */
248
+ private targetOn;
192
249
  /**
193
- * How often this source re-reads a segment's pointer while the segment is being read, in ms: its TTL, or 0 when
194
- * it has no timed refresh (no clock, no registry, or a TTL of 0). The grounded cost report prices the refresh at
195
- * this.
250
+ * Memoize a snapshot as the segment's, forgetting it later if it resolves to null / throws (identity-guarded, so a
251
+ * stale cleanup never clobbers a fresher snapshot). Installing the (pending) snapshot **synchronously** is what lets
252
+ * concurrent callers coalesce onto one in-flight resolve.
196
253
  */
197
- get pointerRefreshMs(): number;
254
+ private install;
255
+ private now;
198
256
  /**
199
257
  * Whether a stream reading from `snap` must resolve its segment again before its next chunk, as a read of one chunk
200
- * would: the snapshot's TTL has lapsed, an invalidation has happened since `epoch`, or the snapshot is no longer the
201
- * one the reader cache holds for the segment (it let it go, or another read installed a newer one). Else nothing can
202
- * have moved it, and the check costs a compare and a lookup.
258
+ * would. In this order: an invalidation has happened since `epoch` (what the store that erased stops on, at once);
259
+ * the TTL of the snapshot's resolution has lapsed; or that resolution is no longer the segment's: another read
260
+ * installed a newer one (a refresh, or a heal), or the resolution cache let it go. With no resolution cache (no timed
261
+ * refresh) the last test is of the snapshot itself, so there the reader cache letting it go is a move, as it is one of
262
+ * the few things that ever moves such a store's read on. With one, it is not. Else nothing can have moved it, and the
263
+ * check costs a compare and a lookup in each cache.
203
264
  */
204
265
  private mayHaveMoved;
205
266
  private expired;
206
267
  /**
207
- * The segment's current generation number — the engine keys its chunk cache by this so a generation bump
208
- * is observed instead of serving a stale decoded chunk. Served from the (TTL-refreshed)
209
- * snapshot, so no extra backend read within the TTL window. `null` if the segment has no committed generation.
268
+ * The segment's current generation number, as its snapshot's resolution found it: no backend read within the TTL
269
+ * window, and no open of the object, ever. `null` if the segment has no committed generation. A generation swept from
270
+ * under the resolution is met by the read that fetches from it, which heals ({@link withFreshSnapshot}). Since it
271
+ * opens nothing, the only error it meets is the resolution's own, and it fails with that error at once: there is
272
+ * nothing to heal. The reads that can open the object (`listChunkKeys`, `summary`, `currentVersion` and the rest)
273
+ * resolve once more on any `NotFoundError`, a registry's among them, since theirs can be a swept generation.
210
274
  *
211
- * Heals a swept generation exactly like {@link withFreshSnapshot}: a caller that resolves the generation once
212
- * per op, before any chunk fetch, would otherwise fail the whole operation rather than one chunk. The engine
213
- * keys by {@link currentVersion} instead, which heals the same way, and falls back to this for a source that
214
- * cannot report a version. It is spelled out rather than delegated because a lookup like this runs once per
215
- * operand of every `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with
216
- * no backend call at all. Routing it through the generic helper cost ~115 ns/op on that path (a second async
217
- * frame, a per-call closure, and an `await` on a plain number) for a race that fires only during a
218
- * concurrent sweep. One retry, then propagate — same contract, same eviction rule.
275
+ * It is spelled out rather than delegated because a lookup like this runs once per operand of every
276
+ * `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with no backend call at all.
277
+ * Routing it through the generic helper cost ~115 ns/op on that path (a second async frame, a per-call closure, and
278
+ * an `await` on a plain number).
219
279
  */
220
280
  currentGeneration(ref: SegmentRef): Promise<number | null>;
221
281
  /**
222
- * Forget everything derived from `ref`: the resolved snapshot and the open reader behind it (and with the
223
- * reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
282
+ * Forget everything derived from `ref`: its resolution, the resolved snapshot and the open reader behind it (and with
283
+ * the reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
224
284
  * resolves the segment again.
225
285
  *
226
286
  * Needed because the TTL refresh only ever answers "has the pointer moved?", and every *destructive* verb —
@@ -231,24 +291,34 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
231
291
  */
232
292
  invalidate(ref: SegmentRef): void;
233
293
  /**
234
- * `<generation>#<object>` for a registry-less source, or `<generation>:<row token>#<object>` with one: the generation
235
- * the segment's snapshot resolved, and the object its reader opened, by that object's size and footer checksum in
236
- * base 36, which the open read anyway. The token moves on every row write, so an unrelated write (a `setRetention`)
237
- * costs the segment's decoded chunks once — bounded, and on an admin path. Two objects do not share a version
238
- * string, as far as their sizes and footer checksums tell them apart, whichever way a number came to name a second
239
- * one: a name purged and loaded again (a new row, or with no registry the same bare number), a number taken again
240
- * within one row once its object was deleted, or an object put back under its key from outside the library. That
241
- * holds however long after its row read a reader opens: a snapshot that a `count()` resolved from the row opens its
242
- * reader only when a later read needs it, and the object under the key may have changed in between. The version is
243
- * opaque: compare two for equality.
294
+ * `<generation>#<object>` for a registry-less source, or `<generation>:<pointerId>#<object>` with one: the generation
295
+ * the segment's snapshot resolved, and the object a reader of it opens, by that object's size and footer checksum in
296
+ * base 36. The row's `pointerId` moves only with a write that names a field a read resolves through, at any value, so
297
+ * a lease or a policy write (a `setRetention`) leaves the version, the open reader and the decoded chunks as they were.
298
+ * Two objects do not share a version string, as far as their sizes and footer checksums tell them apart, whichever way
299
+ * a number came to name a second one: a name purged and loaded again (a new row, or with no registry the same bare
300
+ * number), a number taken again within one row once its object was deleted, or an object put back under its key from
301
+ * outside the library. The version is opaque: compare two for equality.
244
302
  *
245
- * This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its
246
- * chunk cache, so it heals a swept generation exactly as {@link currentGeneration} does: unhealed, a cold read
247
- * racing a publish and a `keep: 0` sweep failed the whole operation with `NotFoundError`, where the same race
248
- * on a chunk fetch heals. Spelled out rather than delegated for the reason `currentGeneration` gives: it is
249
- * almost always served from the cached snapshot with no backend call at all. One retry, then propagate.
303
+ * Where the row's summary names the object, the version is answered from the resolution, with no open: every live
304
+ * open of the generation is held to that object ({@link openLive}), refused as a move otherwise, so the one object any
305
+ * reader of this resolution can name is the one the summary names, and a version named from it is the one that
306
+ * reader's would be. So a `has()` whose chunk is in the chunk cache opens nothing, after the reader cache let
307
+ * the segment go included (a read that needs the index opens the object for it, and on an encrypted segment the
308
+ * key is unwrapped to read the row's sealed summary); an object under the number that is not the row's is met by the read that fetches from it,
309
+ * which refuses it and heals. Where the row has no summary it can use, or there is no registry, nothing names the
310
+ * object but the object: the reader is opened, and the version names what it opened, however long after its row read
311
+ * that is.
312
+ *
313
+ * This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its chunk
314
+ * cache, so an open it makes heals a swept generation as a read does: unhealed, a cold read racing a publish and a
315
+ * `keep: 0` sweep failed the whole operation with `NotFoundError`, where the same race on a chunk fetch heals. Spelled
316
+ * out rather than delegated for the reason `currentGeneration` gives: it is almost always served from the cached
317
+ * snapshot with no backend call at all. One retry, then propagate.
250
318
  */
251
319
  currentVersion(ref: SegmentRef): Promise<string | null>;
320
+ /** {@link currentVersion} of one snapshot: named by the row's summary when it can be, else by the reader, opened. */
321
+ private versionOn;
252
322
  /**
253
323
  * Resolve the segment **once** and report what a pin should hold: the generation, and the version that
254
324
  * identifies those exact bytes. `null` when the segment resolves to no generation — there is nothing to pin,
@@ -282,7 +352,7 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
282
352
  private pinOnce;
283
353
  /**
284
354
  * Where a pinned reader of one object is memoised. With a registry that is its version, which names the object:
285
- * a name purged and loaded again gets a new row, and with it a new token. Without one the version is the bare
355
+ * a name purged and loaded again gets a new row, and with it a new `pointerId`. Without one the version is the bare
286
356
  * generation number, which two incarnations share, so the object's fingerprint is added to tell them apart.
287
357
  */
288
358
  private pinnedKey;
@@ -314,9 +384,10 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
314
384
  * **And the object is checked**, when the caller says which one it pinned. A generation number is not an
315
385
  * identity (invariant 1): once a name is purged and loaded again, its new segment starts again at generation 0,
316
386
  * so a pin of the old one would open the new one's object as its own, and read it beside the chunks it had
317
- * already cached from the old one. The row's token cannot tell them apart, since it moves on every write, a
318
- * publish included; the object's fingerprint can, and a pin whose object has been replaced fails, as one whose
319
- * generation has been swept does. The memo is keyed so that two pins of one generation number in two
387
+ * already cached from the old one. The row's `pointerId` cannot tell them apart, since it moves with every write
388
+ * that changes what the row resolves to, a publish included, while the pinned object stays what it was; the
389
+ * object's fingerprint can, and a pin whose object has been replaced fails, as one whose generation has been swept
390
+ * does. The memo is keyed so that two pins of one generation number in two
320
391
  * incarnations never share a reader, with a registry or without one ({@link pinnedKey}).
321
392
  */
322
393
  private readerAt;
@@ -343,12 +414,12 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
343
414
  * for {@link getChunkAt}.
344
415
  */
345
416
  cardinalitiesAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<ReadonlyMap<number, number> | null>;
346
- /** Grounded size of a specific generation. `held` as for {@link getChunkAt}. */
417
+ /** Size of a specific generation. `held` as for {@link getChunkAt}. */
347
418
  sizeOfAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<SegmentSize | null>;
348
419
  /**
349
- * Whether the segment is a registered name at all. With a registry that is exactly "a row exists" — including
350
- * a row with no generation yet (minted by `setRetention` before the first load) and a `destroyed` tombstone,
351
- * both of which are names somebody deliberately created. Without a registry there is no registration to
420
+ * Whether the segment is a registered name at all. With a registry: a row that is not a `destroyed` tombstone,
421
+ * including one with no generation yet (minted by `setRetention` before the first load), which somebody deliberately
422
+ * created. A tombstone holds nothing, so it answers `false`. Without a registry there is no registration to
352
423
  * consult, so the honest answer is whether the bucket holds any generation of it.
353
424
  *
354
425
  * Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
@@ -356,30 +427,64 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
356
427
  */
357
428
  exists(ref: SegmentRef): Promise<boolean>;
358
429
  /**
359
- * Evict a snapshot we just failed to read from — matched by the snapshot itself, so a concurrent call's
360
- * fresher snapshot is never clobbered. The identity guard is the whole point: `install` replaces the entry
361
- * wholesale, so comparing anything else would drop a good snapshot on the floor.
430
+ * Evict a snapshot we just failed to read from, and the resolution it was built on — each matched by identity, so a
431
+ * concurrent call's fresher snapshot or resolution is never clobbered. The identity guard is the whole point: `install`
432
+ * and `remember` replace an entry wholesale, so comparing anything else would drop a good one on the floor. The
433
+ * resolution goes too, or the read's retry would build its snapshot on it again and reopen the generation it just
434
+ * found gone; and so does a snapshot the reader cache holds on that same resolution, which another read built once the
435
+ * cache had let `snap` go, or the retry would open that one.
362
436
  */
363
437
  private dropStale;
364
438
  /** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
365
439
  private openForTarget;
366
440
  private openGeneration;
367
441
  /**
368
- * Open the generation a resolution found, and hold the row's summary against the object while it is open anyway:
369
- * the same count and metadata, or the summary is not believed again in this process ({@link distrusted}). This
370
- * costs no request; a mismatch is not an error, since the object is the truth and this read has it.
442
+ * Open the generation a resolution found, and hold the row's summary against the object: the object must be the one
443
+ * the summary names by its fingerprint, and is refused before its index is opened otherwise, as another object under
444
+ * the generation's number ({@link NotFoundError}, so the read resolves the row again, as for a swept generation); and
445
+ * it should hold the same count and metadata, or the summary is not believed again in this process
446
+ * ({@link distrusted}). Neither check costs a request. A row with no summary it can use names no object, and its
447
+ * generation's object is opened as it is.
448
+ *
449
+ * An encrypted generation's tail read is sent first, beside the key, because the key (a keystore call, a network
450
+ * round trip for a KMS keystore) and the tail are independent: only the index needs the key, and it comes out of the
451
+ * tail. The row's summary of an encrypted generation is sealed under the same key, so it is read after the key. The
452
+ * open then holds the footer to the summary and decrypts, as an open that waited for the key does, and nothing of the
453
+ * tail is read before then. The key is awaited first, so a keystore fault reaches the caller before a storage fault,
454
+ * and the request already sent is not cancelled: it is billed, and its answer or failure is dropped when the key
455
+ * fails.
456
+ *
457
+ * A cleartext generation's tail read is sized from the `tailBytes` of that summary ({@link sizedTailBytes}), so a
458
+ * generation whose index is longer than the default tail comes in the one read, with no range read for the index
459
+ * after it. Only a summary this process still believes sizes a read ({@link trustedSummary}): one found to disagree
460
+ * with its object is not used again, so a wrong number costs one extra read for the process, not for each open. An
461
+ * encrypted generation is not sized: its summary is sealed under the key, so sizing from it would send the tail read
462
+ * after the key, which the early read above avoids where no earlier step has already waited for the key (a cold
463
+ * `has()` has).
371
464
  */
372
465
  private openLive;
373
466
  private distrustKey;
374
467
  /** What the row says of the generation, unless it is a summary this process has found to disagree with its object. */
375
468
  private trustedSummary;
376
469
  /**
377
- * The current generation + its DEK wrappings: the registry's authoritative record (one strong read), or a
378
- * `list` scan for the max generation when there's no registry (a registry-less source can only read cleartext
379
- * — there's nowhere a wrapped DEK could live). The row's summary rides along, to be used or not by whoever
380
- * asks.
470
+ * What the segment resolves to now: the fields of the registry's authoritative record a read resolves through (one
471
+ * strong read), or the highest generation a `list` of the bucket finds when there is no registry (a registry-less
472
+ * source can only read cleartext: there is nowhere a wrapped DEK could live). `null` for no generation.
473
+ */
474
+ private resolve;
475
+ /**
476
+ * The segment resolved afresh for a pin, with the row it read, which a lease is written against. Never a kept
477
+ * resolution, and kept nowhere: the generation current right now is the whole promise of a pin.
381
478
  */
382
479
  private resolveLive;
480
+ /**
481
+ * What a snapshot, or a pin, makes of a resolution: its decryption context and what the row's summary says of the
482
+ * generation, each made only when asked for. A snapshot that replaces `prior` with the same wrapped keys keeps the key
483
+ * `prior` unwrapped, so a count polling an encrypted segment asks the keystore once, not once each `genTtlMs`.
484
+ * Different keys (a shred, a re-created name) unwrap afresh, and so does a snapshot with no prior: the resolution
485
+ * holds the wrapped keys, never an unwrapped one.
486
+ */
487
+ private liveOf;
383
488
  /** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
384
489
  private cryptoForRead;
385
490
  getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
@@ -397,6 +502,16 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
397
502
  * {@link getChunks} of a specific generation, the pinned shape read. It yields the chunks without a version: the
398
503
  * caller that pinned the generation knows it. It never moves to another generation; `held` as for
399
504
  * {@link getChunkAt}.
505
+ *
506
+ * Before it hands out a chunk after an {@link invalidate}, it looks its generation up as a read of that chunk alone
507
+ * would. Where the invalidation was of its segment, which forgets the pinned reader, it opens the object again, and
508
+ * fails as that read would, with `NotFoundError` once the object is gone or replaced, or its row is gone or
509
+ * destroyed. An object still there is the one it was reading, since an object is immutable and the open is held to
510
+ * the pin, so the stream carries on with the ranges it holds. Another segment's invalidation costs it nothing, the
511
+ * pin's reader still in the reader cache or not, and with no invalidation since it last looked the check is a
512
+ * compare. The one exception is a store that invalidates more than 1,024 other segments while the stream runs: the
513
+ * record of invalidations then counts the stream's segment as invalidated too, and the stream looks its pin up
514
+ * again, at no cost where the reader cache still holds the pin's reader and with one open where it has let it go.
400
515
  */
401
516
  getChunksAt(ref: SegmentRef, generation: number, keys: readonly number[], held?: PinnedObject, options?: ReadChunksOptions): AsyncGenerator<Omit<ChunkRead, 'version'>>;
402
517
  listChunkKeys(ref: SegmentRef): Promise<number[]>;
@@ -407,16 +522,35 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
407
522
  * it can: the row's summary of the generation it names, on an active row, in the shape the row's keys call for
408
523
  * (a sealed one is opened with the segment's key), and one this process has not found to disagree with its
409
524
  * object. That is one registry read when the snapshot is cold, none when it is warm, and no read of the object.
410
- * With no summary it can use (an older row, one for another generation, one that does not open), it opens the
411
- * generation and answers from the object. A row that names an object that is gone still answers, while a read
412
- * of the object throws: the number is true of the generation the row names.
525
+ * With no summary it can use (a row written without one, one for another generation, one that does not open), it
526
+ * opens the generation and answers from the object. A row that names an object that is gone, or another object
527
+ * under the number, still answers, while a read of the object throws: the number is true of the generation the row
528
+ * names.
413
529
  *
414
530
  * What the row says is not confirmed here: a summary edited to agree with itself by whoever can write the row is
415
531
  * believed until the next time the object is opened, which holds it against the summary.
416
532
  */
417
533
  summary(ref: SegmentRef): Promise<GenerationSummary | null>;
534
+ /**
535
+ * What one resolution of the segment says of its generation: from the row's summary when this process may use it,
536
+ * else from the generation's object, opened. `null` for no generation. One heal, as a read of the object has: a
537
+ * generation swept from under the snapshot, or replaced under its number, is resolved again once.
538
+ */
539
+ private described;
418
540
  /** {@link summary} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
419
541
  summaryAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<GenerationSummary | null>;
542
+ /**
543
+ * The current generation's number, id count and metadata, with its object's size, all from one resolution. The answer
544
+ * comes from the row when it can, as {@link summary}'s does: the row's summary of the generation it names, whose
545
+ * fingerprint carries the object's size. That is one registry read when the snapshot is cold, none when it is warm,
546
+ * and no read of the object; on an encrypted segment the key is unwrapped to open the summary, as for a count. With no
547
+ * summary it can use, it opens the generation and answers from its footer and index, with no payload read: a tail read
548
+ * more when cold. A row that names an object that is gone, or another object, still answers from the row: the figures
549
+ * are the generation's the row names, and `checkConsistency` is what finds such a row.
550
+ */
551
+ stat(ref: SegmentRef): Promise<(GenerationSummary & SegmentSize) | null>;
552
+ /** {@link stat} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
553
+ statAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<(GenerationSummary & SegmentSize) | null>;
420
554
  /**
421
555
  * Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
422
556
  * generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
@@ -424,8 +558,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
424
558
  * caller-visible error, so instead we drop the stale snapshot,
425
559
  * re-resolve `currentGen`, and retry once — the read then serves the newer (committed, immutable) generation,
426
560
  * a monotonic move forward — within one incarnation of the row. A name that was retired and re-created is a
427
- * different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's token
428
- * as its lineage rather than trusting the generation number (invariant 1).
561
+ * different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's
562
+ * `pointerId` as its lineage rather than trusting the generation number (invariants 1 and 2).
429
563
  *
430
564
  * **Resolution and open are inside the retry, not before it.** Resolving `currentGen` and opening that
431
565
  * generation's object are two backend round trips with a gap between them, so the object can be swept after
@@ -439,11 +573,20 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
439
573
  * is the documented outcome rather than a failure.
440
574
  *
441
575
  * **Bounded to exactly two resolve-and-open round trips.** That is a cost contract, not a detail: the retry
442
- * now re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
576
+ * re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
443
577
  * tight loop on a segment whose pointer names an object that is permanently absent — and an N-way
444
578
  * `intersect` would do it N times. Gated in `tests/core/storage-source-heal-open.test.ts` by counting calls.
445
579
  */
446
580
  private withFreshSnapshot;
581
+ /** The live snapshot of `ref` and the reader it is opening, as a read holds them while it resolves. */
582
+ private resolution;
583
+ /**
584
+ * The reader `at` resolves to. A caller that retries has the resolution and open run through its `retry`; a failed
585
+ * one forgets itself, so each try resolves anew and `at` is moved to the snapshot that try resolved.
586
+ */
587
+ private resolveReader;
588
+ /** The options a chunk stream is read with: the caller's tuning, its retry for each range, and this source's clock. */
589
+ private readOptionsOf;
447
590
  /**
448
591
  * Whether `err`, raised by a read through the snapshot's reader, came from an object that is no longer the one the
449
592
  * reader opened. A generation number can be taken again once its object is deleted (an erasure removes the
@@ -456,393 +599,3 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
456
599
  */
457
600
  private replacedUnder;
458
601
  }
459
- /**
460
- * Write one immutable generation from in-memory bitmaps. Chunks are
461
- * sorted ascending and empty bitmaps skipped (empty chunks are never stored). Returns the driver's
462
- * `{ size, sha256 }` for the written object. Pass `options.crypto` to AES-256-GCM-encrypt the generation
463
- * (built from the segment's DEK, with associated data bound to `(segment, generation)`).
464
- */
465
- export declare function writeCrbmGeneration(driver: IStorageDriver, key: GenKey, chunks: Iterable<{
466
- chunkKey: number;
467
- bitmap: CodecBitmap;
468
- }>, options?: {
469
- crypto?: CrbmCrypto;
470
- clock?: Yielder;
471
- metadata?: GenerationMetadata;
472
- }): Promise<{
473
- size: number;
474
- sha256: string;
475
- fingerprint: string;
476
- }>;
477
- /**
478
- * Whether the object under `key` is provably the one `fingerprint` names ({@link CrbmReader.fingerprint}), from one
479
- * read of its footer. False when it is another object, when it is gone, and when the read fails in any way: a caller
480
- * that deletes on `true` deletes nothing it cannot prove is its own.
481
- */
482
- export declare function holdsObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
483
- /** Whether the object under `key` says it is encrypted, from one read of its footer, with no key. */
484
- export declare function objectIsEncrypted(storage: IStorageDriver, key: GenKey): Promise<boolean>;
485
- /**
486
- * What one tail read of the object under `key` tells a rollback about it: whether the footer says it is encrypted, and,
487
- * when the object and the key agree (an encrypted object opened with `crypto`, a cleartext one with none), the reader
488
- * the open gives, which holds its count and metadata. No reader when they disagree, which the caller refuses as a
489
- * target the row cannot use, and none for an encrypted object when the caller has no key. A footer that fails its own
490
- * checks, an object another generation's, and an index or metadata that do not open are an {@link IntegrityError}.
491
- */
492
- export declare function openRollbackTarget(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined): Promise<{
493
- encrypted: boolean;
494
- reader: CrbmReader | undefined;
495
- }>;
496
- /** What {@link writeCrbmGenerationStream} wrote: the driver's `{ size, sha256 }` + a tally of the generation. */
497
- export interface StreamWriteResult {
498
- readonly size: number;
499
- readonly sha256: string;
500
- /** Non-empty chunk keys written, in ascending order (for the post-write verify, since the stream is consumed). */
501
- readonly chunkKeys: number[];
502
- /** Total ids written. */
503
- readonly cardinality: number;
504
- /** The fingerprint of what this call wrote ({@link CrbmReader.fingerprint}), which tells it from another object under its key. */
505
- readonly fingerprint: string;
506
- }
507
- /**
508
- * Streaming variant of {@link writeCrbmGeneration} for a **constant-memory** rewrite (the erasure rewrite, a
509
- * sorted load): consumes an **already-ascending** async stream of `{ chunkKey, bitmap }`, feeding each to the
510
- * codec and freeing it, instead of materializing the whole generation. Paired with a streaming storage sink (S3
511
- * multipart / LocalFs temp file), the writer itself holds ~one chunk + one part. Returns a tally so the caller can verify
512
- * the re-opened object without re-iterating the (now-consumed) stream. Input **must** be ascending by `chunkKey`
513
- * — the codec rejects an out-of-order chunk; empty bitmaps are skipped.
514
- */
515
- export declare function writeCrbmGenerationStream(driver: IStorageDriver, key: GenKey, chunks: AsyncIterable<{
516
- chunkKey: number;
517
- bitmap: CodecBitmap;
518
- }>, options?: {
519
- crypto?: CrbmCrypto;
520
- clock?: Yielder;
521
- metadata?: GenerationMetadata;
522
- }): Promise<StreamWriteResult>;
523
- /**
524
- * Point a segment's registry `currentGen` at `key.generation` — the publish step that makes a freshly-written
525
- * generation the authoritative latest (so registry-aware readers see it). **Forward-only and idempotent:**
526
- * if the registry has no row it creates one; if it's already at/ahead of `key.generation` it's a no-op (an
527
- * out-of-order/duplicate publish never regresses the pointer); otherwise it advances via compare-and-swap,
528
- * retrying a few times under contention. Separated from the Storage write so callers can publish atomically
529
- * after the immutable object is durable (write-then-publish).
530
- *
531
- * **Throws** rather than returning, in two cases, because each would otherwise create a state no reader can
532
- * recover from: a `destroyed` (crypto-shredded) row, where advancing the pointer would name an object encrypted
533
- * under a shredded key; and an advance carrying **new** `wrappedDeks`, which would either strand that key or make
534
- * the row advertise encryption over cleartext generations. Contention that never clears throws
535
- * {@link WriteConflictError} after the retries.
536
- *
537
- * **`expectFrom` turns forward-only into read-modify-write.** Forward-only is the right rule for a writer whose
538
- * content does not depend on what was current — a load computes its ids upstream, so publishing over a newer
539
- * generation loses nothing the loader knew about. It is the WRONG rule for a writer that *derived* its content
540
- * from a particular generation: the erasure rewrite streams generation `from` and clears one bit, so publishing
541
- * it over a newer generation silently discards whatever that generation added. Passing `expectFrom` makes the
542
- * publish land only while the pointer is still exactly there, and return `false` otherwise, so the caller can
543
- * report `superseded` and re-derive. The compare-and-swap itself carries the token read in the same iteration,
544
- * so the check and the write see the same row.
545
- *
546
- * **`expectToken` fences the LINEAGE, and a derived writer needs both.** A generation number identifies a
547
- * generation only within one incarnation of a name: the numbering restarts at `0` once the row is purged and
548
- * the bucket empty, so a name that is retired and re-created has a *different* segment wearing the *same*
549
- * `currentGen`. `expectFrom` alone matched it — and an erasure rewrite then published one incarnation's content
550
- * over another's, deleted the live objects with its `keep: 0` collection, and returned `erased: true`. The row's
551
- * OCC token is the identity that survives this: a shipped registry's later `create` gets a token never issued before
552
- * under that name, but for a collision of probability 2^-128 per pair of incarnations, so no token is reused across them.
553
- *
554
- * Pass the token read alongside `expectFrom` and the publish lands only on the same row it was derived from.
555
- * The check is deliberately **conservative**: the token also changes on writes that are not supersessions at
556
- * all (a `setRetention`, a due-index reindex), so such a write makes a derived publish report `superseded` and
557
- * the caller re-derive. That costs a re-run on a rare, unrelated write; the alternative costs a segment.
558
- *
559
- * **The driver sends a registry write once, and an unanswered one is reconciled by its effect.** A `create` or
560
- * `compareAndSwap` that fails with {@link TransientError} (a throttle, a lost response, a timeout) may or may not
561
- * have landed, and may still be in flight. The publish reads the row once more and decides from what it holds:
562
- *
563
- * - the pointer names `key.generation`, on the incarnation the write was made against (the row's `createdAt`), and
564
- * `holdsOwnObject` proves the object under it the caller's: the write landed, and the publish returns `true`. Only
565
- * the writer of a write-once object publishes its number, and a number is never taken while an object holds it;
566
- * - the row is still the one the write was made against (the same token, or still no row): the write may land yet. The
567
- * publish then waits, on the injected clock, and sends a **new** write from the row it just read: a request of its
568
- * own, not a replay of the driver's, carrying the same version, so under the registry's fence at most one of the two
569
- * lands and the other meets a conflict, which the next read settles. At most {@link UNANSWERED_RESENDS} follow, after
570
- * a wait under 500 ms, then 1 s, then 2 s (full jitter when an `rng` is given); each is settled the same way. Still
571
- * unanswered, or with no clock to wait on, this throws the registry's own `TransientError` and writes nothing more.
572
- * A caller must not delete its object on it;
573
- * - anything else: the row has moved past the state the write was conditioned on, so that write can never land, and
574
- * the publish goes on as after a lost race, from the row it just read. A pointer at `key.generation` that is not over
575
- * the caller's own object, or is on another incarnation, is a lost race too and answers `false`, never an already
576
- * current re-publish.
577
- *
578
- * A failed read throws `TransientError` too. The same check of the pointer runs after a {@link WriteConflictError},
579
- * so a write that landed and then met itself (a registry whose client re-sent it) is not taken for a lost race.
580
- *
581
- * **A `summary` goes in the same write that moves the pointer**, so a reader that sees this generation as current
582
- * sees the count and metadata that describe it, and every attempt sends the same one. It must be the shape the row's
583
- * keys call for: sealed for an object written with a key, clear for one written without.
584
- */
585
- export declare function publishGenerationKept(registry: IRegistryDriver, key: GenKey, options?: {
586
- wrappedDeks?: readonly WrappedDek[];
587
- expectFrom?: number;
588
- expectToken?: Token;
589
- /**
590
- * The row `expectToken` names, as the caller read it. With it, a row whose token has moved but which differs from
591
- * this one only in its leases (a reader took or released one) does not refuse the publish: it goes on against the
592
- * fresh row, after a jittered wait and without the caller redoing its work, up to a bound. Any other difference
593
- * refuses as ever. Without it, a moved token refuses.
594
- */
595
- expectRow?: RegistryRecord | null;
596
- /**
597
- * Publish only while the segment still has **no registry row**.
598
- *
599
- * The fence for a caller whose decision rests on the row being ABSENT: every load that found no row sets it,
600
- * guarded or not. `expectFrom` and `expectToken` cannot express it: both compare against a value read from a
601
- * row, so when there was no row there is nothing to compare and both are simply omitted — leaving the publish a
602
- * bare forward-only advance that happily lands over whatever appeared in the meantime.
603
- *
604
- * Without it, a guarded load into a segment that did not exist yet would judge "no row", pass its empty and
605
- * `minRetained` bounds vacuously, and land an EMPTY generation over the ids a concurrent writer had just loaded,
606
- * reporting `published: true`. An unguarded load would move the pointer over another first load's generation or
607
- * onto a row `setRetention` made, and, where an erasure had deleted its object above that row's pointer, onto an
608
- * object that is gone. Materialising into a destination that does not exist yet is the ordinary first run of a
609
- * pipeline, so the `*Into` verbs reach this as often as `loadSegment` does.
610
- */
611
- expectAbsent?: boolean;
612
- /**
613
- * The generation's summary (see {@link RegistrySummary}), written by the write that moves the pointer to it. It
614
- * names `key.generation`. Absent, the row is left with none for this generation.
615
- */
616
- summary?: RegistrySummary;
617
- /**
618
- * The segment's row as the caller already read it (`null`: it found none). The first attempt acts on it
619
- * instead of reading the row again, which is sound because that attempt writes only under the row's own
620
- * fence: its compare-and-swap carries this row's token, and with no row it creates create-only, so a row
621
- * that changed or appeared since makes the write lose and the next attempt reads the row afresh. Only the
622
- * idempotent "already current" answer writes nothing, so a reused row never gives it.
623
- */
624
- row?: RegistryRecord | null;
625
- /**
626
- * The object was written without encryption. Its publish is refused, with {@link KeyUnavailableError}, onto a row
627
- * that carries key material: a reader of that segment opens every generation with its key, and a later
628
- * `destroySegment` would attest that shredding it made a readable object unreadable. A caller decides its key
629
- * from the row it read before the write, so this closes the window in which another writer creates the
630
- * segment encrypted meanwhile.
631
- */
632
- cleartext?: boolean;
633
- /**
634
- * Whether the object under `key` is the one the caller wrote, and throws when it cannot tell. Asked only when a
635
- * registry write failed and the row then names `key.generation`, to tell this publish's write from another
636
- * writer's at the same number. Every caller that writes the object passes it. Without it, a write that ended
637
- * without an answer over a row that names `key.generation` cannot be settled: the publish throws
638
- * {@link TransientError} and the caller keeps its object.
639
- */
640
- holdsOwnObject?: () => Promise<boolean>;
641
- /**
642
- * What a publish waits on before it sends a fresh compare-and-swap, after a write that ended without an answer and
643
- * left the row as it was. Without a `sleep`, such a write throws at once instead of being sent again.
644
- */
645
- clock?: Yielder;
646
- /** Spreads that wait (full jitter) so concurrent publishers do not retry together. Without it the wait is the bound. */
647
- rng?: Rng;
648
- /**
649
- * Called once for each registry write that ended without an answer, before the row is read back. A caller whose
650
- * publish then answers `false` can say that it cannot tell whether its own write landed first: the row may have
651
- * held its generation for a while before another writer moved on.
652
- */
653
- onUnanswered?: () => void;
654
- /**
655
- * The clock that judges whether a lease has ended. With it, a publish over a row that holds ended leases drops
656
- * them in the write that moves the pointer, which rewrites the row anyway. Without it, leases are left as they are.
657
- */
658
- leasesNow?: () => number;
659
- /**
660
- * Clear the row's leases in the write that moves the pointer: the caller's collection deletes every generation
661
- * below the new pointer whatever a lease says, as a subject erasure's does.
662
- */
663
- clearLeases?: boolean;
664
- /**
665
- * How many generations below the new pointer the caller keeps. With it, the write that moves the pointer also
666
- * records the window in the row (`keptGens`: the newest `keep` of the row's own list and the generation that was
667
- * current), derived from the very row the write is made against, and the result names the generations the window
668
- * dropped. Without it, or above the most a row records, the row records none, and a collection lists.
669
- */
670
- keep?: number;
671
- }): Promise<PublishResult>;
672
- /** What a publish did: whether the pointer is at the generation, and what the write recorded in the row. */
673
- export interface PublishResult {
674
- readonly published: boolean;
675
- /**
676
- * What the write that moved the pointer recorded: the list, the generations the window dropped, and the new row's
677
- * token. Absent when no write of this call is known to have landed (the pointer was already there, or the outcome
678
- * was settled without knowing which write it was), and a collection then lists.
679
- */
680
- readonly kept?: PublishedKept;
681
- }
682
- /**
683
- * {@link publishGenerationKept}'s answer to "did the pointer move", for a caller with no use for what it recorded:
684
- * a publish with no `keep` leaves the row recording no list.
685
- */
686
- export declare function publishGeneration(registry: IRegistryDriver, key: GenKey, options?: Parameters<typeof publishGenerationKept>[2]): Promise<boolean>;
687
- /**
688
- * Whether the object under `key` is the one `fingerprint` names, from one read of its footer: false when it is another
689
- * object or gone. Unlike {@link holdsObject}, a read that cannot tell throws, for a caller that must not take "could
690
- * not look" for "not this object".
691
- */
692
- export declare function provesOwnObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
693
- /** What a bulk-load wrote — the driver's `{ size, sha256 }` plus a summary of the built generation. */
694
- export interface BulkLoadResult {
695
- /** Bytes written to the Storage object. */
696
- readonly size: number;
697
- /** The driver's content hash of the object. */
698
- readonly sha256: string;
699
- /** Distinct non-empty chunks written (≤ 65536). */
700
- readonly chunkCount: number;
701
- /**
702
- * The DEK freshly minted for this generation, wrapped — present only when the segment is encrypted AND this
703
- * call minted one (an existing segment reuses its key, which is already on the row).
704
- *
705
- * Returned so a caller that deliberately does **not** pass `registry` here — writing now and publishing later,
706
- * which is how a load applies a guard while the old generation is still authoritative — can carry the key onto
707
- * its own `publishGeneration`. Without it that publish stores no DEK and the generation it makes current is
708
- * unreadable: the bytes are encrypted with a key nothing recorded.
709
- */
710
- readonly wrappedDeks?: readonly WrappedDek[];
711
- /** Whether the object was written encrypted: under the segment's key, or under one this call minted. */
712
- readonly encrypted: boolean;
713
- /** The written object's fingerprint, which tells it from an object stored later under the same key. */
714
- readonly fingerprint: string;
715
- /** Total distinct ids in the generation (post-dedup). */
716
- readonly cardinality: number;
717
- /**
718
- * The summary of what was written, for the write that makes it current: its id count and metadata, sealed under the
719
- * segment's key when the object was written with one. A caller that defers the publish (`publish: false`) passes it
720
- * to {@link publishGeneration}.
721
- */
722
- readonly summary: RegistrySummary;
723
- /**
724
- * Whether this generation is now the segment's **current** one — `undefined` when no `registry` was wired, so
725
- * there is no pointer and no publish step to report on.
726
- *
727
- * `false` means the object is durable but a **concurrent writer published a higher generation first**, so this
728
- * one is an orphan that no reader will ever resolve. The write succeeded and the *load* did not: whatever this
729
- * call was asked to make the segment contain, the segment does not contain. That is not a detail a caller can
730
- * be left to infer — a `*Into` materialisation that reported the generation it wrote as "the destination's new
731
- * current generation" on this path would state a plain untruth — so the flag is on the result and the verbs that
732
- * promise a published generation check it.
733
- */
734
- readonly becameCurrent?: boolean;
735
- }
736
- /**
737
- * **Bulk-load** a whole immutable generation from a flat id source — the batch "seed/rebuild" entry point.
738
- * Unlike {@link writeCrbmGeneration} (which takes pre-grouped bitmaps), this
739
- * consumes an arbitrary, **unsorted** stream of u32 ids (sync or async) and routes each into its chunk's
740
- * Roaring bitmap as it goes. The input is consumed lazily and ids dedup on insert, so memory is bounded by
741
- * the **built generation** — the in-memory Roaring representation of the *distinct* set (≤ one bitmap per
742
- * non-empty chunk) — not by the input length: you can stream a billion duplicate-heavy ids and hold only the
743
- * distinct result. Note this is `O(distinct set)`, **not** window-bounded like {@link SegmentEngine.intersect}:
744
- * bulk-load holds the whole generation in RAM, which suits batch seed/rebuild jobs but not an
745
- * unbounded-cardinality stream (a segment larger than RAM needs external-merge / pre-sorted input — later phase).
746
- *
747
- * Each id must be an integer in `[0, 2³²)` ({@link splitId} throws {@link ValidationError} otherwise) — the
748
- * source is consumed lazily, so a bad id aborts mid-stream without writing a partial object (the driver
749
- * commits only after the callback resolves). An empty source writes a valid empty generation.
750
- *
751
- * Writes a fresh full snapshot of a segment. The caller picks the generation number in `key` — `nextGeneration`
752
- * computes one from the registry and the bucket, as `nextLoadGeneration` does for a load — and re-using an existing
753
- * generation throws
754
- * {@link WriteConflictError} (write-once). Without a registry a `StorageChunkSource` serves the **highest**
755
- * generation present, so a too-high number silently shadows real data; with one, `publishGeneration` decides.
756
- */
757
- export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenKey, ids: Iterable<number> | AsyncIterable<number> | DecodedLoadInput | ChunkLoadInput, options?: {
758
- registry?: IRegistryDriver;
759
- keystore?: IKeystore;
760
- requireEncryption?: boolean;
761
- audit?: IAuditSink;
762
- /** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
763
- codec?: CodecInterface;
764
- /**
765
- * Injected clock. Supplying one makes bulk-load **cooperative**: it yields the event loop periodically so a
766
- * long load does not stall everything else on the process. Without it the load still completes, just
767
- * without yielding — which is the pre-existing behaviour, kept so this is purely additive.
768
- *
769
- * `@cloudbitmaps/roaring` supplies a real clock by default, so flavor users get cooperative behaviour with
770
- * no wiring. `core/` cannot default it: it is timer-free by lint, which is exactly why waiting goes through
771
- * this seam rather than `setTimeout`.
772
- */
773
- clock?: Clock;
774
- /**
775
- * Write the object but do **not** advance the pointer (default: publish when a `registry` is wired).
776
- *
777
- * For a caller that has to inspect what it wrote before deciding whether it should become current — a
778
- * guarded {@link loadSegment} is the one in-tree case — because the check is only meaningful while the old
779
- * generation is still authoritative. The `registry` is still required for an encrypted segment and still
780
- * consulted: it is where an existing segment's DEK lives, and reusing that key is not optional. The wrapped
781
- * DEK comes back on the result so the deferred publish can store it.
782
- *
783
- * A caller that defers the publish owns what it wrote: an unpublished object sits ABOVE `currentGen`, where
784
- * generation collection deliberately never looks, so nothing reclaims it until a later generation above it is
785
- * current.
786
- */
787
- publish?: boolean;
788
- /**
789
- * The generation's metadata (see {@link GenerationMetadata}), written into the object and carried in the summary.
790
- * The caller has checked it against the metadata rules; the writer checks it again, and refuses a record that breaks
791
- * one before a byte is written.
792
- */
793
- metadata?: GenerationMetadata;
794
- /**
795
- * The segment's row as the caller already read it (`null`: it found none), for a caller that defers the
796
- * publish (`publish: false`) and fences it on that same row. A present cleartext row is then used as read,
797
- * rather than read again: a row that changes in between (a publish, a drop, a purge and re-create) makes the
798
- * fenced publish lose, so the object is never published on the strength of the stale read. A row read as
799
- * absent, or one carrying key material, is read again after the ids, as with no row passed: a first load must
800
- * see a row another writer created meanwhile, and an encrypted segment's key is unwrapped only from a row read
801
- * after the ids, so a segment shredded while they streamed is refused before its key is used.
802
- */
803
- row?: RegistryRecord | null;
804
- }): Promise<BulkLoadResult>;
805
- /**
806
- * What a load has already started when it calls {@link bulkLoadAhead}: its number, and the segment's data key, each
807
- * still in flight so their round trips overlap the encoding. Both are joined where the write needs them, and neither
808
- * may be left to reject unobserved.
809
- */
810
- export interface LoadAhead {
811
- /** The number the generation takes (see {@link nextLoadGeneration}); joined just before the object is written. */
812
- readonly generation: PromiseLike<number>;
813
- /**
814
- * The key a caller already asked the keystore to unwrap, with the wrappings it asked about. Used only when the
815
- * row the write reads after the ids carries exactly these wrappings; otherwise the write unwraps for itself.
816
- */
817
- readonly unwrapped?: {
818
- readonly wrapped: readonly WrappedDek[];
819
- readonly aead: PromiseLike<Aead>;
820
- };
821
- }
822
- /** What a load with `requireEncryption` meets on a segment that already has a cleartext generation. */
823
- export declare function cleartextUnderRequiredEncryption(ref: SegmentRef, currentGen: number): ValidationError;
824
- /**
825
- * {@link bulkLoadCrbmGeneration} for a load whose number, and perhaps its key, are still being fetched: the same
826
- * options, with the same meaning, and the same writes. The number is joined only where the object is written.
827
- */
828
- export declare function bulkLoadAhead(driver: IStorageDriver, ref: SegmentRef, ahead: LoadAhead, ids: Parameters<typeof bulkLoadCrbmGeneration>[2], options?: NonNullable<Parameters<typeof bulkLoadCrbmGeneration>[3]>): Promise<BulkLoadResult>;
829
- /**
830
- * Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
831
- * The reader the write paths use to re-read what they wrote, and the erasure rewrite uses to stream the old
832
- * generation; the engine's read path goes through {@link CrbmStorageChunkSource} instead, which caches these.
833
- */
834
- export declare function openGenerationReader(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, options?: Omit<CrbmReaderOptions, 'crypto'>): Promise<CrbmReader>;
835
- /**
836
- * Re-open a freshly written generation and assert it round-trips exactly what was streamed into it: the same
837
- * per-chunk key set *and* the same total cardinality (on top of the codec's own per-chunk CRC + footer checks), and the
838
- * metadata it was given. `expected` is the streaming writer's tally (the stream is consumed, so it can't be
839
- * re-iterated) — the key-set comparison catches a dropped/extra chunk that a cardinality-only check could miss when
840
- * two errors cancel out. The count and the metadata are what the generation's row summary is about to claim, so the
841
- * summary is held to the object it describes before it is published.
842
- * Throws {@link IntegrityError}: the object is on disk but must not be published.
843
- */
844
- export declare function verifyGeneration(storage: IStorageDriver, key: GenKey, expected: {
845
- readonly chunkKeys: readonly number[];
846
- readonly cardinality: number;
847
- readonly metadata?: GenerationMetadata | undefined;
848
- }, crypto: CrbmCrypto | undefined): Promise<void>;