@cloudbitmaps/core 0.19.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -0
- package/dist/{chunk-RSLVTRJY.js → chunk-IGLIZBR7.js} +982 -281
- package/dist/chunk-IGLIZBR7.js.map +7 -0
- package/dist/core/absent-operands.d.ts +8 -0
- package/dist/core/audit.d.ts +67 -4
- package/dist/core/bit-route.d.ts +2 -1
- package/dist/core/blob.d.ts +12 -1
- package/dist/core/bulk-load.d.ts +148 -0
- package/dist/core/cas-retry.d.ts +27 -0
- package/dist/core/check.d.ts +22 -0
- package/dist/core/chunk-checks.d.ts +6 -1
- package/dist/core/chunk-window.d.ts +11 -4
- package/dist/core/combine-expr.d.ts +1 -3
- package/dist/core/combine-feed.d.ts +2 -16
- package/dist/core/combine-many.d.ts +21 -15
- package/dist/core/concurrency.d.ts +8 -0
- package/dist/core/consistency.d.ts +76 -18
- package/dist/core/crbm/fingerprint.d.ts +23 -0
- package/dist/core/crbm/format.d.ts +2 -5
- package/dist/core/crbm/frame.d.ts +10 -0
- package/dist/core/crbm/plan-reads.d.ts +9 -3
- package/dist/core/crbm/reader.d.ts +51 -12
- package/dist/core/crbm/varint.d.ts +2 -1
- package/dist/core/crbm/writer.d.ts +5 -1
- package/dist/core/crbm-open.d.ts +154 -0
- package/dist/core/crbm-storage-source.d.ts +243 -490
- package/dist/core/crbm-write.d.ts +75 -0
- package/dist/core/crypto.d.ts +6 -1
- package/dist/core/engine.d.ts +239 -47
- package/dist/core/erase-id.d.ts +97 -27
- package/dist/core/erasure.d.ts +4 -3
- package/dist/core/errors.d.ts +8 -2
- package/dist/core/generation-gc.d.ts +131 -16
- package/dist/core/generation-list.d.ts +11 -0
- package/dist/core/id-window.d.ts +46 -0
- package/dist/core/invalidation-scope.d.ts +77 -0
- package/dist/core/known-generations-shape.d.ts +13 -0
- package/dist/core/known-generations.d.ts +147 -0
- package/dist/core/leases.d.ts +9 -1
- package/dist/core/ledger-seed.d.ts +17 -0
- package/dist/core/load.d.ts +17 -36
- package/dist/core/lru.d.ts +2 -15
- package/dist/core/metrics.d.ts +9 -7
- package/dist/core/order.d.ts +2 -0
- package/dist/core/pinned-storage-source.d.ts +7 -7
- package/dist/core/pointer-id.d.ts +19 -0
- package/dist/core/ports.d.ts +210 -45
- package/dist/core/publish.d.ts +207 -0
- package/dist/core/reader-defaults.d.ts +14 -3
- package/dist/core/registry-scan.d.ts +0 -2
- package/dist/core/retention-sweep.d.ts +69 -65
- package/dist/core/retention.d.ts +1 -1
- package/dist/core/retry.d.ts +12 -19
- package/dist/core/rollback.d.ts +62 -32
- package/dist/core/summary.d.ts +74 -25
- package/dist/core/token.d.ts +13 -8
- package/dist/core/validate.d.ts +2 -0
- package/dist/driver-kit.d.ts +6 -1
- package/dist/driver-kit.js +90 -10
- package/dist/driver-kit.js.map +3 -3
- package/dist/drivers/_shared/bytes.d.ts +6 -0
- package/dist/drivers/_shared/faults.d.ts +15 -0
- package/dist/drivers/_shared/object-keys.d.ts +18 -0
- package/dist/drivers/_shared/object-registry-keys.d.ts +0 -2
- package/dist/drivers/_shared/object-registry.d.ts +3 -1
- package/dist/drivers/_shared/options.d.ts +19 -0
- package/dist/drivers/_shared/registry.d.ts +31 -19
- package/dist/drivers/localfs/exact-case.d.ts +0 -3
- package/dist/drivers/localfs/fs-util.d.ts +22 -0
- package/dist/drivers/localfs/registry.d.ts +10 -4
- package/dist/drivers/localfs/storage.d.ts +16 -8
- package/dist/drivers/memory.d.ts +14 -9
- package/dist/drivers/retry/retrying-chunk-source.d.ts +1 -5
- package/dist/export/index.d.ts +16 -0
- package/dist/index.d.ts +2 -5
- package/dist/index.js +4303 -2938
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
- package/dist/chunk-RSLVTRJY.js.map +0 -7
- package/dist/core/cost.d.ts +0 -351
- package/dist/core/kept-generations.d.ts +0 -40
|
@@ -1,14 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import type {
|
|
4
|
-
import type { Clock, Rng } from './determinism.js';
|
|
5
|
-
import type { PublishedKept } from './generation-gc.js';
|
|
6
|
-
import type { Aead, CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
|
|
7
|
-
import type { ChunkRef, ChunkRead, ReadChunksOptions, StorageChunkSource, GenerationMetadata, GenerationSummary, GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, RegistrySummary, SegmentRef, SegmentSize, Token } from './ports.js';
|
|
8
|
-
import { CrbmReader } from './crbm/reader.js';
|
|
1
|
+
import type { Clock } from './determinism.js';
|
|
2
|
+
import type { IKeystore } from './crypto.js';
|
|
3
|
+
import type { ChunkRef, ChunkRead, ReadChunksOptions, StorageChunkSource, GenerationSummary, IStorageDriver, IRegistryDriver, RegistryRecord, SegmentRef, SegmentSize, Token } from './ports.js';
|
|
9
4
|
import type { CrbmReaderOptions } from './crbm/reader.js';
|
|
10
|
-
import type { CodecBitmap, CodecInterface } from './codec.js';
|
|
11
|
-
import { ChunkLoadInput, DecodedLoadInput } from './load-input.js';
|
|
12
5
|
export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
|
|
13
6
|
/**
|
|
14
7
|
* Optional {@link IRegistryDriver}. When provided, the current generation is resolved via the registry's
|
|
@@ -45,7 +38,7 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
|
|
|
45
38
|
* How long (ms) a resolved `currentGen` is trusted before the next read re-resolves it (default 2000) — the
|
|
46
39
|
* bound on read staleness after a load publishes: a reader may serve the prior generation for up to this long,
|
|
47
40
|
* then converges. Lazy (checked on read — no timer); ≤ one cheap registry read per segment per window, and a
|
|
48
|
-
* new {@link CrbmReader} is opened only when the
|
|
41
|
+
* new {@link CrbmReader} is opened only when the row's pointer was renewed. `0` turns the refresh off, as a
|
|
49
42
|
* missing clock or registry does: {@link clock} says what that changes, and what it does not.
|
|
50
43
|
*/
|
|
51
44
|
readonly currentGenTtlMs?: number;
|
|
@@ -53,7 +46,9 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
|
|
|
53
46
|
* Hard ceiling on how many segments' readers (each holding a fully-parsed `.crbm` index) are cached at once
|
|
54
47
|
* (default 1024) — a memory bound for a long-running server. Past it, the least-recently-used
|
|
55
48
|
* segment's reader is evicted; the next read of an evicted segment re-opens it (one cheap tail GET, since
|
|
56
|
-
* generations are immutable). Raise it for a big cache working set of small segments.
|
|
49
|
+
* generations are immutable). Raise it for a big cache working set of small segments, and raise {@link maxOpenIndexBytes} with it: each open reader counts about 3.75 KiB there, so past about 17,000 tiny readers the byte ceiling binds first. A source with a timed refresh
|
|
50
|
+
* also keeps up to 8 times this many segments' resolutions apart from their readers, for the TTL: the re-open then
|
|
51
|
+
* reads no row, and comes only when a read needs the reader, for the index or for a chunk.
|
|
57
52
|
*/
|
|
58
53
|
readonly maxOpenSegments?: number;
|
|
59
54
|
/**
|
|
@@ -62,31 +57,33 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
|
|
|
62
57
|
* **second half of that memory bound**. `maxOpenSegments` alone bounds by *count*, but a wide/dense segment's
|
|
63
58
|
* parsed index can reach about 1.3 MB, so 1024 wide indices could pin over a GB and blow a small heap (e.g. a 128 MB
|
|
64
59
|
* Lambda) while the count is nominally "in bounds". This caps the summed {@link CrbmReader.retainedBytes} (the
|
|
65
|
-
* parsed index, any metadata a generation carries, and any kept chunk bytes)
|
|
60
|
+
* parsed index, any metadata a generation carries, and any kept chunk bytes) plus a fixed 3.75 KiB for the objects each open reader
|
|
61
|
+
* holds around them, across cached readers, so that many tiny segments are bounded too; the least-recently-used reader is evicted once the total would exceed it — whichever
|
|
66
62
|
* of the count/byte bounds binds first. A reader whose open read the whole object keeps a copy of its chunk region when
|
|
67
63
|
* that region is at most this ceiling divided by {@link maxOpenSegments} (64 KiB by default) — its fair share — so a small generation's
|
|
68
64
|
* chunks are served from memory with no further request, and a bigger one is read by range. Only a source with a timed
|
|
69
65
|
* pointer refresh ({@link currentGenTtlMs} above 0, a registry and a clock) keeps chunk bytes. Lower it for memory-tight deployments with wide segments; a single
|
|
70
|
-
* segment whose
|
|
66
|
+
* segment whose weight alone exceeds the budget is still cached (it can't be shrunk) but nothing else alongside. A
|
|
67
|
+
* sixteenth of it bounds the resolutions a source with a timed refresh keeps besides (4 MiB by default).
|
|
71
68
|
*/
|
|
72
69
|
readonly maxOpenIndexBytes?: number;
|
|
73
70
|
}
|
|
74
71
|
/**
|
|
75
72
|
* What a pin holds of the object it pinned: the version its cache entries are keyed by, and the object's
|
|
76
|
-
* fingerprint, which a pinned read checks each time it opens the generation. `pinGeneration` always records
|
|
77
|
-
*
|
|
73
|
+
* fingerprint, which a pinned read checks each time it opens the generation. `pinGeneration` always records both,
|
|
74
|
+
* and a pinned read given one without both as strings throws a `ValidationError`.
|
|
78
75
|
*/
|
|
79
76
|
export interface PinnedObject {
|
|
80
77
|
readonly version: string;
|
|
81
|
-
readonly fingerprint
|
|
78
|
+
readonly fingerprint: string;
|
|
82
79
|
}
|
|
83
80
|
/**
|
|
84
|
-
* What a live version's resolution named, without the object's fingerprint: `<generation>:<
|
|
81
|
+
* What a live version's resolution named, without the object's fingerprint: `<generation>:<pointerId>`, or
|
|
85
82
|
* `<generation>` with no registry. For a caller that compares a live version, from
|
|
86
|
-
* {@link CrbmStorageChunkSource.currentVersion} or a chunk of its `getChunks`, with a row it read
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
83
|
+
* {@link CrbmStorageChunkSource.currentVersion} or a chunk of its `getChunks`, with a row it read. A live version always
|
|
84
|
+
* ends with the suffix, which holds no `#`, so the match starts at its last `#` and a `pointerId` holding `#` of its own
|
|
85
|
+
* is kept whole. Known limit: a string with no suffix, such as a pin's version or one from another source, loses its end
|
|
86
|
+
* when its `pointerId` itself ends in `#<base 36>.<base 36>`; no caller passes one.
|
|
90
87
|
*/
|
|
91
88
|
export declare function rowVersionOf(version: string): string;
|
|
92
89
|
/** What a pin that leases its generation does between reading the row and opening the object. */
|
|
@@ -96,6 +93,15 @@ export interface PinLease {
|
|
|
96
93
|
readonly token: Token;
|
|
97
94
|
} | 'moved'>;
|
|
98
95
|
}
|
|
96
|
+
/**
|
|
97
|
+
* The fixed heap an open reader's snapshot holds beyond {@link CrbmReader.retainedBytes}: the snapshot, the reader, its
|
|
98
|
+
* five index arrays as objects, and the closures and promises that tie them. It is added to a reader's weight in the
|
|
99
|
+
* reader cache, so that the byte ceiling bounds many tiny segments as it does a few wide ones. Measured as the heap that
|
|
100
|
+
* stayed (full collections until stable) per segment after a source opened 2,000 one-chunk segments, minus the 20 bytes
|
|
101
|
+
* of index the weight already counts: about 3.7 KiB on Node 24, rounded up to 3.75 KiB.
|
|
102
|
+
* `tests/core/reader-cache-heap.test.ts` holds the counted weight to that heap.
|
|
103
|
+
*/
|
|
104
|
+
export declare const OPEN_READER_BYTES = 3840;
|
|
99
105
|
export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
100
106
|
private readonly driver;
|
|
101
107
|
/**
|
|
@@ -112,6 +118,21 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
112
118
|
* steady-state memory bound; re-opening an evicted segment is one cheap tail GET.
|
|
113
119
|
*/
|
|
114
120
|
private readonly snapshots;
|
|
121
|
+
/**
|
|
122
|
+
* Each segment's resolution, kept apart from its reader, by segment: what a read of its row found, for the TTL from
|
|
123
|
+
* the instant that read was sent, whether or not the reader cache still holds the segment's reader. So letting a reader
|
|
124
|
+
* go moves nothing: the next read builds a snapshot on the resolution, reads no row, and opens the object when it needs
|
|
125
|
+
* the index or a chunk the chunk cache does not hold (a version, a count and a stat from the row's summary need
|
|
126
|
+
* neither). It holds the resolved fields only ({@link Resolved}): the wrapped keys, never an unwrapped one, so a
|
|
127
|
+
* snapshot built on it unwraps the key again through the keystore. It never holds a resolution that found no
|
|
128
|
+
* generation: the next read of such a segment reads the row.
|
|
129
|
+
*
|
|
130
|
+
* Only a source with a timed refresh has one: without it, letting a reader go is one of the few things that ever moves
|
|
131
|
+
* a read on, and keeping the resolution would keep it forever. **Bounded** by count and by bytes, both derived from
|
|
132
|
+
* the reader cache's: {@link RESOLUTIONS_PER_OPEN_SEGMENT} times its count bound, and its byte bound over
|
|
133
|
+
* {@link RESOLUTION_BYTES_DIVISOR}. An entry heavier than the byte bound is kept alone, as the reader cache keeps one.
|
|
134
|
+
*/
|
|
135
|
+
private readonly resolutions;
|
|
115
136
|
/**
|
|
116
137
|
* Pins ({@link heldKey}) whose object has been found replaced: another object is under its generation's key. Kept,
|
|
117
138
|
* and bounded as the readers are, so that a pin's later chunk reads and any reopen fail at once rather than pay for
|
|
@@ -133,7 +154,12 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
133
154
|
* counts every segment's invalidations: a read that waited through another segment's only costs a later read the
|
|
134
155
|
* check again.
|
|
135
156
|
*/
|
|
136
|
-
private invalidations;
|
|
157
|
+
private get invalidations();
|
|
158
|
+
/**
|
|
159
|
+
* Which segments {@link invalidate} has been called for, the latest invalidation of up to 1,024 of them: a pinned
|
|
160
|
+
* stream asks it whether its own segment was invalidated since it last looked.
|
|
161
|
+
*/
|
|
162
|
+
private readonly invalidated;
|
|
137
163
|
/**
|
|
138
164
|
* Generations whose row summary this process found to disagree with their object, by segment and generation. A
|
|
139
165
|
* summary here is not used again: a count of that generation reads the object. Bounded as the readers are, so a
|
|
@@ -141,6 +167,13 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
141
167
|
* the disagreement again.
|
|
142
168
|
*/
|
|
143
169
|
private readonly distrusted;
|
|
170
|
+
/**
|
|
171
|
+
* The readers held to the row's summary: by a live open ({@link openLive}), or by a live pin that shared one. A pin
|
|
172
|
+
* that finds one memoised under its version shares it as it is: the version names the row's `pointerId`, and every
|
|
173
|
+
* write of the summary renews it, so the summary it was held to is the one the row has now. Any other memoised reader
|
|
174
|
+
* (a `pinAt`'s, held to the fingerprint its caller named) is held to the row's summary before a live pin shares it.
|
|
175
|
+
*/
|
|
176
|
+
private readonly rowChecked;
|
|
144
177
|
private readonly registry;
|
|
145
178
|
private readonly keystore;
|
|
146
179
|
private readonly requireEncryption;
|
|
@@ -151,8 +184,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
151
184
|
private readonly currentGenTtlMs;
|
|
152
185
|
/**
|
|
153
186
|
* Whether the segment's pointer is re-read on a timer: it needs a clock (the TTL), a registry (the cheap row read; with
|
|
154
|
-
* none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes
|
|
155
|
-
*
|
|
187
|
+
* none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes and the TTL
|
|
188
|
+
* lapse, so it is decided once.
|
|
156
189
|
*/
|
|
157
190
|
private readonly timedRefresh;
|
|
158
191
|
constructor(driver: IStorageDriver, options?: CrbmStorageChunkSourceOptions);
|
|
@@ -164,63 +197,90 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
164
197
|
private readonly keys;
|
|
165
198
|
private keyOf;
|
|
166
199
|
/**
|
|
167
|
-
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window
|
|
168
|
-
*
|
|
200
|
+
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window: the snapshot the reader cache holds,
|
|
201
|
+
* while its resolution is inside the TTL. With a timed refresh, a segment whose reader the cache let go is served a
|
|
202
|
+
* snapshot built on its kept resolution, while that is inside the TTL: no row is read, and the reader opens only when a
|
|
203
|
+
* read needs one. Otherwise the pointer is read now. The snapshot holds the resolved target, and its reader once a read
|
|
204
|
+
* has asked for one.
|
|
169
205
|
*/
|
|
170
206
|
private liveSnapshot;
|
|
171
|
-
/** The current resolved reader for a segment, refreshing on the TTL. */
|
|
172
|
-
private resolvedReader;
|
|
173
207
|
/**
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
*
|
|
208
|
+
* Read the segment's pointer now. The read is sent before this returns, and the TTL counts from this instant: a read
|
|
209
|
+
* that answers late does not lengthen the life of what it found. The row read is always fresh, so what a snapshot on
|
|
210
|
+
* it says of the generation (its summary included) is what the row says now, never `prior`'s.
|
|
211
|
+
*
|
|
212
|
+
* A read that fails with a transient fault answers what `prior` found, when it found a generation, and is asked again
|
|
213
|
+
* soon ({@link retrySoon}): `prior` is the resolution of the snapshot the reader cache still holds, so an outage of the
|
|
214
|
+
* registry keeps that snapshot's generation, and its reader and unwrapped key where it has them; one resolved without
|
|
215
|
+
* an open, for a count or a generation lookup, unwraps the key from `prior`'s wrapped keys when a read first needs
|
|
216
|
+
* it. The outage ends shortly after the registry answers.
|
|
217
|
+
* That answer is the snapshot's alone ({@link Resolution.rodeOut}): it is not kept as the segment's resolution, and a
|
|
218
|
+
* snapshot built on it after the reader cache let that one go fails with the fault. With no `prior` the fault
|
|
219
|
+
* fails the read, as a cold resolve's does. If `prior` found none or failed, the pointer is read once more rather than
|
|
220
|
+
* a dead resolution re-armed (else the segment reads empty for a whole TTL window). Anything else — an access denial, a corrupt row, a
|
|
221
|
+
* registry that answers NotFound — fails the read exactly as a cold resolve of the segment would, and the resolution
|
|
222
|
+
* and the snapshot on it are forgotten, so a reader and the key it unwrapped do not outlive a refresh that could not
|
|
223
|
+
* be trusted.
|
|
224
|
+
*/
|
|
225
|
+
private resolveNow;
|
|
226
|
+
/**
|
|
227
|
+
* A refresh failed transiently and the prior resolution keeps serving: make it lapse after {@link REFRESH_RETRY_MS}
|
|
228
|
+
* (never longer than the TTL) rather than after a whole TTL, so the next read after that asks the registry again. It
|
|
229
|
+
* costs no read of its own; it only changes when the next refresh happens.
|
|
177
230
|
*/
|
|
178
231
|
private retrySoon;
|
|
179
232
|
/**
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
*
|
|
233
|
+
* Keep `resolution` as the segment's, from the instant its read was sent, so the reads that come while it is in flight
|
|
234
|
+
* share it, and a read after the reader cache let the segment go builds on it. Its weight is reported once it lands,
|
|
235
|
+
* and one that finds no generation, or fails, is forgotten. Both are identity-guarded, as the reader cache's are, and
|
|
236
|
+
* nothing is installed when the read lands: an `invalidate` while it is in flight, or a newer resolution, is never
|
|
237
|
+
* undone by its answer.
|
|
183
238
|
*/
|
|
184
|
-
private
|
|
239
|
+
private remember;
|
|
185
240
|
/**
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
*
|
|
241
|
+
* A live snapshot on `resolution`. `before` is the snapshot it replaces, while the reader cache still holds one: its
|
|
242
|
+
* reader is kept if the resolution finds the same generation, `pointerId` and object ({@link Snapshot.reuse}), and its
|
|
243
|
+
* unwrapped key if the same wrapped keys. A snapshot built after the reader cache let the segment go has neither, so
|
|
244
|
+
* its open asks the keystore again: a key is held only while a reader of the segment is.
|
|
189
245
|
*/
|
|
190
|
-
private
|
|
191
|
-
|
|
246
|
+
private snapshotOn;
|
|
247
|
+
/** What the snapshot on `resolution` that replaces `before` reads: see {@link snapshotOn}. */
|
|
248
|
+
private targetOn;
|
|
192
249
|
/**
|
|
193
|
-
*
|
|
194
|
-
*
|
|
195
|
-
*
|
|
250
|
+
* Memoize a snapshot as the segment's, forgetting it later if it resolves to null / throws (identity-guarded, so a
|
|
251
|
+
* stale cleanup never clobbers a fresher snapshot). Installing the (pending) snapshot **synchronously** is what lets
|
|
252
|
+
* concurrent callers coalesce onto one in-flight resolve.
|
|
196
253
|
*/
|
|
197
|
-
|
|
254
|
+
private install;
|
|
255
|
+
private now;
|
|
198
256
|
/**
|
|
199
257
|
* Whether a stream reading from `snap` must resolve its segment again before its next chunk, as a read of one chunk
|
|
200
|
-
* would
|
|
201
|
-
*
|
|
202
|
-
*
|
|
258
|
+
* would. In this order: an invalidation has happened since `epoch` (what the store that erased stops on, at once);
|
|
259
|
+
* the TTL of the snapshot's resolution has lapsed; or that resolution is no longer the segment's: another read
|
|
260
|
+
* installed a newer one (a refresh, or a heal), or the resolution cache let it go. With no resolution cache (no timed
|
|
261
|
+
* refresh) the last test is of the snapshot itself, so there the reader cache letting it go is a move, as it is one of
|
|
262
|
+
* the few things that ever moves such a store's read on. With one, it is not. Else nothing can have moved it, and the
|
|
263
|
+
* check costs a compare and a lookup in each cache.
|
|
203
264
|
*/
|
|
204
265
|
private mayHaveMoved;
|
|
205
266
|
private expired;
|
|
206
267
|
/**
|
|
207
|
-
* The segment's current generation number
|
|
208
|
-
*
|
|
209
|
-
*
|
|
268
|
+
* The segment's current generation number, as its snapshot's resolution found it: no backend read within the TTL
|
|
269
|
+
* window, and no open of the object, ever. `null` if the segment has no committed generation. A generation swept from
|
|
270
|
+
* under the resolution is met by the read that fetches from it, which heals ({@link withFreshSnapshot}). Since it
|
|
271
|
+
* opens nothing, the only error it meets is the resolution's own, and it fails with that error at once: there is
|
|
272
|
+
* nothing to heal. The reads that can open the object (`listChunkKeys`, `summary`, `currentVersion` and the rest)
|
|
273
|
+
* resolve once more on any `NotFoundError`, a registry's among them, since theirs can be a swept generation.
|
|
210
274
|
*
|
|
211
|
-
*
|
|
212
|
-
*
|
|
213
|
-
*
|
|
214
|
-
*
|
|
215
|
-
* operand of every `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with
|
|
216
|
-
* no backend call at all. Routing it through the generic helper cost ~115 ns/op on that path (a second async
|
|
217
|
-
* frame, a per-call closure, and an `await` on a plain number) for a race that fires only during a
|
|
218
|
-
* concurrent sweep. One retry, then propagate — same contract, same eviction rule.
|
|
275
|
+
* It is spelled out rather than delegated because a lookup like this runs once per operand of every
|
|
276
|
+
* `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with no backend call at all.
|
|
277
|
+
* Routing it through the generic helper cost ~115 ns/op on that path (a second async frame, a per-call closure, and
|
|
278
|
+
* an `await` on a plain number).
|
|
219
279
|
*/
|
|
220
280
|
currentGeneration(ref: SegmentRef): Promise<number | null>;
|
|
221
281
|
/**
|
|
222
|
-
* Forget everything derived from `ref`: the resolved snapshot and the open reader behind it (and with
|
|
223
|
-
* reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
|
|
282
|
+
* Forget everything derived from `ref`: its resolution, the resolved snapshot and the open reader behind it (and with
|
|
283
|
+
* the reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
|
|
224
284
|
* resolves the segment again.
|
|
225
285
|
*
|
|
226
286
|
* Needed because the TTL refresh only ever answers "has the pointer moved?", and every *destructive* verb —
|
|
@@ -231,24 +291,34 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
231
291
|
*/
|
|
232
292
|
invalidate(ref: SegmentRef): void;
|
|
233
293
|
/**
|
|
234
|
-
* `<generation>#<object>` for a registry-less source, or `<generation>:<
|
|
235
|
-
* the segment's snapshot resolved, and the object
|
|
236
|
-
* base 36
|
|
237
|
-
*
|
|
238
|
-
* string, as far as their sizes and footer checksums tell them apart, whichever way
|
|
239
|
-
* one: a name purged and loaded again (a new row, or with no registry the same bare
|
|
240
|
-
* within one row once its object was deleted, or an object put back under its key from
|
|
241
|
-
*
|
|
242
|
-
* reader only when a later read needs it, and the object under the key may have changed in between. The version is
|
|
243
|
-
* opaque: compare two for equality.
|
|
294
|
+
* `<generation>#<object>` for a registry-less source, or `<generation>:<pointerId>#<object>` with one: the generation
|
|
295
|
+
* the segment's snapshot resolved, and the object a reader of it opens, by that object's size and footer checksum in
|
|
296
|
+
* base 36. The row's `pointerId` moves only with a write that names a field a read resolves through, at any value, so
|
|
297
|
+
* a lease or a policy write (a `setRetention`) leaves the version, the open reader and the decoded chunks as they were.
|
|
298
|
+
* Two objects do not share a version string, as far as their sizes and footer checksums tell them apart, whichever way
|
|
299
|
+
* a number came to name a second one: a name purged and loaded again (a new row, or with no registry the same bare
|
|
300
|
+
* number), a number taken again within one row once its object was deleted, or an object put back under its key from
|
|
301
|
+
* outside the library. The version is opaque: compare two for equality.
|
|
244
302
|
*
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
*
|
|
248
|
-
*
|
|
249
|
-
*
|
|
303
|
+
* Where the row's summary names the object, the version is answered from the resolution, with no open: every live
|
|
304
|
+
* open of the generation is held to that object ({@link openLive}), refused as a move otherwise, so the one object any
|
|
305
|
+
* reader of this resolution can name is the one the summary names, and a version named from it is the one that
|
|
306
|
+
* reader's would be. So a `has()` whose chunk is in the chunk cache opens nothing, after the reader cache let
|
|
307
|
+
* the segment go included (a read that needs the index opens the object for it, and on an encrypted segment the
|
|
308
|
+
* key is unwrapped to read the row's sealed summary); an object under the number that is not the row's is met by the read that fetches from it,
|
|
309
|
+
* which refuses it and heals. Where the row has no summary it can use, or there is no registry, nothing names the
|
|
310
|
+
* object but the object: the reader is opened, and the version names what it opened, however long after its row read
|
|
311
|
+
* that is.
|
|
312
|
+
*
|
|
313
|
+
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its chunk
|
|
314
|
+
* cache, so an open it makes heals a swept generation as a read does: unhealed, a cold read racing a publish and a
|
|
315
|
+
* `keep: 0` sweep failed the whole operation with `NotFoundError`, where the same race on a chunk fetch heals. Spelled
|
|
316
|
+
* out rather than delegated for the reason `currentGeneration` gives: it is almost always served from the cached
|
|
317
|
+
* snapshot with no backend call at all. One retry, then propagate.
|
|
250
318
|
*/
|
|
251
319
|
currentVersion(ref: SegmentRef): Promise<string | null>;
|
|
320
|
+
/** {@link currentVersion} of one snapshot: named by the row's summary when it can be, else by the reader, opened. */
|
|
321
|
+
private versionOn;
|
|
252
322
|
/**
|
|
253
323
|
* Resolve the segment **once** and report what a pin should hold: the generation, and the version that
|
|
254
324
|
* identifies those exact bytes. `null` when the segment resolves to no generation — there is nothing to pin,
|
|
@@ -282,7 +352,7 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
282
352
|
private pinOnce;
|
|
283
353
|
/**
|
|
284
354
|
* Where a pinned reader of one object is memoised. With a registry that is its version, which names the object:
|
|
285
|
-
* a name purged and loaded again gets a new row, and with it a new
|
|
355
|
+
* a name purged and loaded again gets a new row, and with it a new `pointerId`. Without one the version is the bare
|
|
286
356
|
* generation number, which two incarnations share, so the object's fingerprint is added to tell them apart.
|
|
287
357
|
*/
|
|
288
358
|
private pinnedKey;
|
|
@@ -314,9 +384,10 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
314
384
|
* **And the object is checked**, when the caller says which one it pinned. A generation number is not an
|
|
315
385
|
* identity (invariant 1): once a name is purged and loaded again, its new segment starts again at generation 0,
|
|
316
386
|
* so a pin of the old one would open the new one's object as its own, and read it beside the chunks it had
|
|
317
|
-
* already cached from the old one. The row's
|
|
318
|
-
*
|
|
319
|
-
*
|
|
387
|
+
* already cached from the old one. The row's `pointerId` cannot tell them apart, since it moves with every write
|
|
388
|
+
* that changes what the row resolves to, a publish included, while the pinned object stays what it was; the
|
|
389
|
+
* object's fingerprint can, and a pin whose object has been replaced fails, as one whose generation has been swept
|
|
390
|
+
* does. The memo is keyed so that two pins of one generation number in two
|
|
320
391
|
* incarnations never share a reader, with a registry or without one ({@link pinnedKey}).
|
|
321
392
|
*/
|
|
322
393
|
private readerAt;
|
|
@@ -343,12 +414,12 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
343
414
|
* for {@link getChunkAt}.
|
|
344
415
|
*/
|
|
345
416
|
cardinalitiesAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<ReadonlyMap<number, number> | null>;
|
|
346
|
-
/**
|
|
417
|
+
/** Size of a specific generation. `held` as for {@link getChunkAt}. */
|
|
347
418
|
sizeOfAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<SegmentSize | null>;
|
|
348
419
|
/**
|
|
349
|
-
* Whether the segment is a registered name at all. With a registry that is
|
|
350
|
-
*
|
|
351
|
-
*
|
|
420
|
+
* Whether the segment is a registered name at all. With a registry: a row that is not a `destroyed` tombstone,
|
|
421
|
+
* including one with no generation yet (minted by `setRetention` before the first load), which somebody deliberately
|
|
422
|
+
* created. A tombstone holds nothing, so it answers `false`. Without a registry there is no registration to
|
|
352
423
|
* consult, so the honest answer is whether the bucket holds any generation of it.
|
|
353
424
|
*
|
|
354
425
|
* Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
|
|
@@ -356,30 +427,64 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
356
427
|
*/
|
|
357
428
|
exists(ref: SegmentRef): Promise<boolean>;
|
|
358
429
|
/**
|
|
359
|
-
* Evict a snapshot we just failed to read from — matched by
|
|
360
|
-
* fresher snapshot is never clobbered. The identity guard is the whole point: `install`
|
|
361
|
-
* wholesale, so comparing anything else would drop a good
|
|
430
|
+
* Evict a snapshot we just failed to read from, and the resolution it was built on — each matched by identity, so a
|
|
431
|
+
* concurrent call's fresher snapshot or resolution is never clobbered. The identity guard is the whole point: `install`
|
|
432
|
+
* and `remember` replace an entry wholesale, so comparing anything else would drop a good one on the floor. The
|
|
433
|
+
* resolution goes too, or the read's retry would build its snapshot on it again and reopen the generation it just
|
|
434
|
+
* found gone; and so does a snapshot the reader cache holds on that same resolution, which another read built once the
|
|
435
|
+
* cache had let `snap` go, or the retry would open that one.
|
|
362
436
|
*/
|
|
363
437
|
private dropStale;
|
|
364
438
|
/** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
|
|
365
439
|
private openForTarget;
|
|
366
440
|
private openGeneration;
|
|
367
441
|
/**
|
|
368
|
-
* Open the generation a resolution found, and hold the row's summary against the object
|
|
369
|
-
* the
|
|
370
|
-
*
|
|
442
|
+
* Open the generation a resolution found, and hold the row's summary against the object: the object must be the one
|
|
443
|
+
* the summary names by its fingerprint, and is refused before its index is opened otherwise, as another object under
|
|
444
|
+
* the generation's number ({@link NotFoundError}, so the read resolves the row again, as for a swept generation); and
|
|
445
|
+
* it should hold the same count and metadata, or the summary is not believed again in this process
|
|
446
|
+
* ({@link distrusted}). Neither check costs a request. A row with no summary it can use names no object, and its
|
|
447
|
+
* generation's object is opened as it is.
|
|
448
|
+
*
|
|
449
|
+
* An encrypted generation's tail read is sent first, beside the key, because the key (a keystore call, a network
|
|
450
|
+
* round trip for a KMS keystore) and the tail are independent: only the index needs the key, and it comes out of the
|
|
451
|
+
* tail. The row's summary of an encrypted generation is sealed under the same key, so it is read after the key. The
|
|
452
|
+
* open then holds the footer to the summary and decrypts, as an open that waited for the key does, and nothing of the
|
|
453
|
+
* tail is read before then. The key is awaited first, so a keystore fault reaches the caller before a storage fault,
|
|
454
|
+
* and the request already sent is not cancelled: it is billed, and its answer or failure is dropped when the key
|
|
455
|
+
* fails.
|
|
456
|
+
*
|
|
457
|
+
* A cleartext generation's tail read is sized from the `tailBytes` of that summary ({@link sizedTailBytes}), so a
|
|
458
|
+
* generation whose index is longer than the default tail comes in the one read, with no range read for the index
|
|
459
|
+
* after it. Only a summary this process still believes sizes a read ({@link trustedSummary}): one found to disagree
|
|
460
|
+
* with its object is not used again, so a wrong number costs one extra read for the process, not for each open. An
|
|
461
|
+
* encrypted generation is not sized: its summary is sealed under the key, so sizing from it would send the tail read
|
|
462
|
+
* after the key, which the early read above avoids where no earlier step has already waited for the key (a cold
|
|
463
|
+
* `has()` has).
|
|
371
464
|
*/
|
|
372
465
|
private openLive;
|
|
373
466
|
private distrustKey;
|
|
374
467
|
/** What the row says of the generation, unless it is a summary this process has found to disagree with its object. */
|
|
375
468
|
private trustedSummary;
|
|
376
469
|
/**
|
|
377
|
-
*
|
|
378
|
-
* `list`
|
|
379
|
-
*
|
|
380
|
-
|
|
470
|
+
* What the segment resolves to now: the fields of the registry's authoritative record a read resolves through (one
|
|
471
|
+
* strong read), or the highest generation a `list` of the bucket finds when there is no registry (a registry-less
|
|
472
|
+
* source can only read cleartext: there is nowhere a wrapped DEK could live). `null` for no generation.
|
|
473
|
+
*/
|
|
474
|
+
private resolve;
|
|
475
|
+
/**
|
|
476
|
+
* The segment resolved afresh for a pin, with the row it read, which a lease is written against. Never a kept
|
|
477
|
+
* resolution, and kept nowhere: the generation current right now is the whole promise of a pin.
|
|
381
478
|
*/
|
|
382
479
|
private resolveLive;
|
|
480
|
+
/**
|
|
481
|
+
* What a snapshot, or a pin, makes of a resolution: its decryption context and what the row's summary says of the
|
|
482
|
+
* generation, each made only when asked for. A snapshot that replaces `prior` with the same wrapped keys keeps the key
|
|
483
|
+
* `prior` unwrapped, so a count polling an encrypted segment asks the keystore once, not once each `genTtlMs`.
|
|
484
|
+
* Different keys (a shred, a re-created name) unwrap afresh, and so does a snapshot with no prior: the resolution
|
|
485
|
+
* holds the wrapped keys, never an unwrapped one.
|
|
486
|
+
*/
|
|
487
|
+
private liveOf;
|
|
383
488
|
/** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
|
|
384
489
|
private cryptoForRead;
|
|
385
490
|
getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
|
|
@@ -397,6 +502,16 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
397
502
|
* {@link getChunks} of a specific generation, the pinned shape read. It yields the chunks without a version: the
|
|
398
503
|
* caller that pinned the generation knows it. It never moves to another generation; `held` as for
|
|
399
504
|
* {@link getChunkAt}.
|
|
505
|
+
*
|
|
506
|
+
* Before it hands out a chunk after an {@link invalidate}, it looks its generation up as a read of that chunk alone
|
|
507
|
+
* would. Where the invalidation was of its segment, which forgets the pinned reader, it opens the object again, and
|
|
508
|
+
* fails as that read would, with `NotFoundError` once the object is gone or replaced, or its row is gone or
|
|
509
|
+
* destroyed. An object still there is the one it was reading, since an object is immutable and the open is held to
|
|
510
|
+
* the pin, so the stream carries on with the ranges it holds. Another segment's invalidation costs it nothing, the
|
|
511
|
+
* pin's reader still in the reader cache or not, and with no invalidation since it last looked the check is a
|
|
512
|
+
* compare. The one exception is a store that invalidates more than 1,024 other segments while the stream runs: the
|
|
513
|
+
* record of invalidations then counts the stream's segment as invalidated too, and the stream looks its pin up
|
|
514
|
+
* again, at no cost where the reader cache still holds the pin's reader and with one open where it has let it go.
|
|
400
515
|
*/
|
|
401
516
|
getChunksAt(ref: SegmentRef, generation: number, keys: readonly number[], held?: PinnedObject, options?: ReadChunksOptions): AsyncGenerator<Omit<ChunkRead, 'version'>>;
|
|
402
517
|
listChunkKeys(ref: SegmentRef): Promise<number[]>;
|
|
@@ -407,16 +522,35 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
407
522
|
* it can: the row's summary of the generation it names, on an active row, in the shape the row's keys call for
|
|
408
523
|
* (a sealed one is opened with the segment's key), and one this process has not found to disagree with its
|
|
409
524
|
* object. That is one registry read when the snapshot is cold, none when it is warm, and no read of the object.
|
|
410
|
-
* With no summary it can use (
|
|
411
|
-
* generation and answers from the object. A row that names an object that is gone
|
|
412
|
-
* of the object throws: the number is true of the generation the row
|
|
525
|
+
* With no summary it can use (a row written without one, one for another generation, one that does not open), it
|
|
526
|
+
* opens the generation and answers from the object. A row that names an object that is gone, or another object
|
|
527
|
+
* under the number, still answers, while a read of the object throws: the number is true of the generation the row
|
|
528
|
+
* names.
|
|
413
529
|
*
|
|
414
530
|
* What the row says is not confirmed here: a summary edited to agree with itself by whoever can write the row is
|
|
415
531
|
* believed until the next time the object is opened, which holds it against the summary.
|
|
416
532
|
*/
|
|
417
533
|
summary(ref: SegmentRef): Promise<GenerationSummary | null>;
|
|
534
|
+
/**
|
|
535
|
+
* What one resolution of the segment says of its generation: from the row's summary when this process may use it,
|
|
536
|
+
* else from the generation's object, opened. `null` for no generation. One heal, as a read of the object has: a
|
|
537
|
+
* generation swept from under the snapshot, or replaced under its number, is resolved again once.
|
|
538
|
+
*/
|
|
539
|
+
private described;
|
|
418
540
|
/** {@link summary} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
419
541
|
summaryAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<GenerationSummary | null>;
|
|
542
|
+
/**
|
|
543
|
+
* The current generation's number, id count and metadata, with its object's size, all from one resolution. The answer
|
|
544
|
+
* comes from the row when it can, as {@link summary}'s does: the row's summary of the generation it names, whose
|
|
545
|
+
* fingerprint carries the object's size. That is one registry read when the snapshot is cold, none when it is warm,
|
|
546
|
+
* and no read of the object; on an encrypted segment the key is unwrapped to open the summary, as for a count. With no
|
|
547
|
+
* summary it can use, it opens the generation and answers from its footer and index, with no payload read: a tail read
|
|
548
|
+
* more when cold. A row that names an object that is gone, or another object, still answers from the row: the figures
|
|
549
|
+
* are the generation's the row names, and `checkConsistency` is what finds such a row.
|
|
550
|
+
*/
|
|
551
|
+
stat(ref: SegmentRef): Promise<(GenerationSummary & SegmentSize) | null>;
|
|
552
|
+
/** {@link stat} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
553
|
+
statAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<(GenerationSummary & SegmentSize) | null>;
|
|
420
554
|
/**
|
|
421
555
|
* Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
|
|
422
556
|
* generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
|
|
@@ -424,8 +558,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
424
558
|
* caller-visible error, so instead we drop the stale snapshot,
|
|
425
559
|
* re-resolve `currentGen`, and retry once — the read then serves the newer (committed, immutable) generation,
|
|
426
560
|
* a monotonic move forward — within one incarnation of the row. A name that was retired and re-created is a
|
|
427
|
-
* different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's
|
|
428
|
-
* as its lineage rather than trusting the generation number (
|
|
561
|
+
* different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's
|
|
562
|
+
* `pointerId` as its lineage rather than trusting the generation number (invariants 1 and 2).
|
|
429
563
|
*
|
|
430
564
|
* **Resolution and open are inside the retry, not before it.** Resolving `currentGen` and opening that
|
|
431
565
|
* generation's object are two backend round trips with a gap between them, so the object can be swept after
|
|
@@ -439,11 +573,20 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
439
573
|
* is the documented outcome rather than a failure.
|
|
440
574
|
*
|
|
441
575
|
* **Bounded to exactly two resolve-and-open round trips.** That is a cost contract, not a detail: the retry
|
|
442
|
-
*
|
|
576
|
+
* re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
|
|
443
577
|
* tight loop on a segment whose pointer names an object that is permanently absent — and an N-way
|
|
444
578
|
* `intersect` would do it N times. Gated in `tests/core/storage-source-heal-open.test.ts` by counting calls.
|
|
445
579
|
*/
|
|
446
580
|
private withFreshSnapshot;
|
|
581
|
+
/** The live snapshot of `ref` and the reader it is opening, as a read holds them while it resolves. */
|
|
582
|
+
private resolution;
|
|
583
|
+
/**
|
|
584
|
+
* The reader `at` resolves to. A caller that retries has the resolution and open run through its `retry`; a failed
|
|
585
|
+
* one forgets itself, so each try resolves anew and `at` is moved to the snapshot that try resolved.
|
|
586
|
+
*/
|
|
587
|
+
private resolveReader;
|
|
588
|
+
/** The options a chunk stream is read with: the caller's tuning, its retry for each range, and this source's clock. */
|
|
589
|
+
private readOptionsOf;
|
|
447
590
|
/**
|
|
448
591
|
* Whether `err`, raised by a read through the snapshot's reader, came from an object that is no longer the one the
|
|
449
592
|
* reader opened. A generation number can be taken again once its object is deleted (an erasure removes the
|
|
@@ -456,393 +599,3 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
456
599
|
*/
|
|
457
600
|
private replacedUnder;
|
|
458
601
|
}
|
|
459
|
-
/**
|
|
460
|
-
* Write one immutable generation from in-memory bitmaps. Chunks are
|
|
461
|
-
* sorted ascending and empty bitmaps skipped (empty chunks are never stored). Returns the driver's
|
|
462
|
-
* `{ size, sha256 }` for the written object. Pass `options.crypto` to AES-256-GCM-encrypt the generation
|
|
463
|
-
* (built from the segment's DEK, with associated data bound to `(segment, generation)`).
|
|
464
|
-
*/
|
|
465
|
-
export declare function writeCrbmGeneration(driver: IStorageDriver, key: GenKey, chunks: Iterable<{
|
|
466
|
-
chunkKey: number;
|
|
467
|
-
bitmap: CodecBitmap;
|
|
468
|
-
}>, options?: {
|
|
469
|
-
crypto?: CrbmCrypto;
|
|
470
|
-
clock?: Yielder;
|
|
471
|
-
metadata?: GenerationMetadata;
|
|
472
|
-
}): Promise<{
|
|
473
|
-
size: number;
|
|
474
|
-
sha256: string;
|
|
475
|
-
fingerprint: string;
|
|
476
|
-
}>;
|
|
477
|
-
/**
|
|
478
|
-
* Whether the object under `key` is provably the one `fingerprint` names ({@link CrbmReader.fingerprint}), from one
|
|
479
|
-
* read of its footer. False when it is another object, when it is gone, and when the read fails in any way: a caller
|
|
480
|
-
* that deletes on `true` deletes nothing it cannot prove is its own.
|
|
481
|
-
*/
|
|
482
|
-
export declare function holdsObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
|
|
483
|
-
/** Whether the object under `key` says it is encrypted, from one read of its footer, with no key. */
|
|
484
|
-
export declare function objectIsEncrypted(storage: IStorageDriver, key: GenKey): Promise<boolean>;
|
|
485
|
-
/**
|
|
486
|
-
* What one tail read of the object under `key` tells a rollback about it: whether the footer says it is encrypted, and,
|
|
487
|
-
* when the object and the key agree (an encrypted object opened with `crypto`, a cleartext one with none), the reader
|
|
488
|
-
* the open gives, which holds its count and metadata. No reader when they disagree, which the caller refuses as a
|
|
489
|
-
* target the row cannot use, and none for an encrypted object when the caller has no key. A footer that fails its own
|
|
490
|
-
* checks, an object another generation's, and an index or metadata that do not open are an {@link IntegrityError}.
|
|
491
|
-
*/
|
|
492
|
-
export declare function openRollbackTarget(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined): Promise<{
|
|
493
|
-
encrypted: boolean;
|
|
494
|
-
reader: CrbmReader | undefined;
|
|
495
|
-
}>;
|
|
496
|
-
/** What {@link writeCrbmGenerationStream} wrote: the driver's `{ size, sha256 }` + a tally of the generation. */
|
|
497
|
-
export interface StreamWriteResult {
|
|
498
|
-
readonly size: number;
|
|
499
|
-
readonly sha256: string;
|
|
500
|
-
/** Non-empty chunk keys written, in ascending order (for the post-write verify, since the stream is consumed). */
|
|
501
|
-
readonly chunkKeys: number[];
|
|
502
|
-
/** Total ids written. */
|
|
503
|
-
readonly cardinality: number;
|
|
504
|
-
/** The fingerprint of what this call wrote ({@link CrbmReader.fingerprint}), which tells it from another object under its key. */
|
|
505
|
-
readonly fingerprint: string;
|
|
506
|
-
}
|
|
507
|
-
/**
|
|
508
|
-
* Streaming variant of {@link writeCrbmGeneration} for a **constant-memory** rewrite (the erasure rewrite, a
|
|
509
|
-
* sorted load): consumes an **already-ascending** async stream of `{ chunkKey, bitmap }`, feeding each to the
|
|
510
|
-
* codec and freeing it, instead of materializing the whole generation. Paired with a streaming storage sink (S3
|
|
511
|
-
* multipart / LocalFs temp file), the writer itself holds ~one chunk + one part. Returns a tally so the caller can verify
|
|
512
|
-
* the re-opened object without re-iterating the (now-consumed) stream. Input **must** be ascending by `chunkKey`
|
|
513
|
-
* — the codec rejects an out-of-order chunk; empty bitmaps are skipped.
|
|
514
|
-
*/
|
|
515
|
-
export declare function writeCrbmGenerationStream(driver: IStorageDriver, key: GenKey, chunks: AsyncIterable<{
|
|
516
|
-
chunkKey: number;
|
|
517
|
-
bitmap: CodecBitmap;
|
|
518
|
-
}>, options?: {
|
|
519
|
-
crypto?: CrbmCrypto;
|
|
520
|
-
clock?: Yielder;
|
|
521
|
-
metadata?: GenerationMetadata;
|
|
522
|
-
}): Promise<StreamWriteResult>;
|
|
523
|
-
/**
|
|
524
|
-
* Point a segment's registry `currentGen` at `key.generation` — the publish step that makes a freshly-written
|
|
525
|
-
* generation the authoritative latest (so registry-aware readers see it). **Forward-only and idempotent:**
|
|
526
|
-
* if the registry has no row it creates one; if it's already at/ahead of `key.generation` it's a no-op (an
|
|
527
|
-
* out-of-order/duplicate publish never regresses the pointer); otherwise it advances via compare-and-swap,
|
|
528
|
-
* retrying a few times under contention. Separated from the Storage write so callers can publish atomically
|
|
529
|
-
* after the immutable object is durable (write-then-publish).
|
|
530
|
-
*
|
|
531
|
-
* **Throws** rather than returning, in two cases, because each would otherwise create a state no reader can
|
|
532
|
-
* recover from: a `destroyed` (crypto-shredded) row, where advancing the pointer would name an object encrypted
|
|
533
|
-
* under a shredded key; and an advance carrying **new** `wrappedDeks`, which would either strand that key or make
|
|
534
|
-
* the row advertise encryption over cleartext generations. Contention that never clears throws
|
|
535
|
-
* {@link WriteConflictError} after the retries.
|
|
536
|
-
*
|
|
537
|
-
* **`expectFrom` turns forward-only into read-modify-write.** Forward-only is the right rule for a writer whose
|
|
538
|
-
* content does not depend on what was current — a load computes its ids upstream, so publishing over a newer
|
|
539
|
-
* generation loses nothing the loader knew about. It is the WRONG rule for a writer that *derived* its content
|
|
540
|
-
* from a particular generation: the erasure rewrite streams generation `from` and clears one bit, so publishing
|
|
541
|
-
* it over a newer generation silently discards whatever that generation added. Passing `expectFrom` makes the
|
|
542
|
-
* publish land only while the pointer is still exactly there, and return `false` otherwise, so the caller can
|
|
543
|
-
* report `superseded` and re-derive. The compare-and-swap itself carries the token read in the same iteration,
|
|
544
|
-
* so the check and the write see the same row.
|
|
545
|
-
*
|
|
546
|
-
* **`expectToken` fences the LINEAGE, and a derived writer needs both.** A generation number identifies a
|
|
547
|
-
* generation only within one incarnation of a name: the numbering restarts at `0` once the row is purged and
|
|
548
|
-
* the bucket empty, so a name that is retired and re-created has a *different* segment wearing the *same*
|
|
549
|
-
* `currentGen`. `expectFrom` alone matched it — and an erasure rewrite then published one incarnation's content
|
|
550
|
-
* over another's, deleted the live objects with its `keep: 0` collection, and returned `erased: true`. The row's
|
|
551
|
-
* OCC token is the identity that survives this: a shipped registry's later `create` gets a token never issued before
|
|
552
|
-
* under that name, but for a collision of probability 2^-128 per pair of incarnations, so no token is reused across them.
|
|
553
|
-
*
|
|
554
|
-
* Pass the token read alongside `expectFrom` and the publish lands only on the same row it was derived from.
|
|
555
|
-
* The check is deliberately **conservative**: the token also changes on writes that are not supersessions at
|
|
556
|
-
* all (a `setRetention`, a due-index reindex), so such a write makes a derived publish report `superseded` and
|
|
557
|
-
* the caller re-derive. That costs a re-run on a rare, unrelated write; the alternative costs a segment.
|
|
558
|
-
*
|
|
559
|
-
* **The driver sends a registry write once, and an unanswered one is reconciled by its effect.** A `create` or
|
|
560
|
-
* `compareAndSwap` that fails with {@link TransientError} (a throttle, a lost response, a timeout) may or may not
|
|
561
|
-
* have landed, and may still be in flight. The publish reads the row once more and decides from what it holds:
|
|
562
|
-
*
|
|
563
|
-
* - the pointer names `key.generation`, on the incarnation the write was made against (the row's `createdAt`), and
|
|
564
|
-
* `holdsOwnObject` proves the object under it the caller's: the write landed, and the publish returns `true`. Only
|
|
565
|
-
* the writer of a write-once object publishes its number, and a number is never taken while an object holds it;
|
|
566
|
-
* - the row is still the one the write was made against (the same token, or still no row): the write may land yet. The
|
|
567
|
-
* publish then waits, on the injected clock, and sends a **new** write from the row it just read: a request of its
|
|
568
|
-
* own, not a replay of the driver's, carrying the same version, so under the registry's fence at most one of the two
|
|
569
|
-
* lands and the other meets a conflict, which the next read settles. At most {@link UNANSWERED_RESENDS} follow, after
|
|
570
|
-
* a wait under 500 ms, then 1 s, then 2 s (full jitter when an `rng` is given); each is settled the same way. Still
|
|
571
|
-
* unanswered, or with no clock to wait on, this throws the registry's own `TransientError` and writes nothing more.
|
|
572
|
-
* A caller must not delete its object on it;
|
|
573
|
-
* - anything else: the row has moved past the state the write was conditioned on, so that write can never land, and
|
|
574
|
-
* the publish goes on as after a lost race, from the row it just read. A pointer at `key.generation` that is not over
|
|
575
|
-
* the caller's own object, or is on another incarnation, is a lost race too and answers `false`, never an already
|
|
576
|
-
* current re-publish.
|
|
577
|
-
*
|
|
578
|
-
* A failed read throws `TransientError` too. The same check of the pointer runs after a {@link WriteConflictError},
|
|
579
|
-
* so a write that landed and then met itself (a registry whose client re-sent it) is not taken for a lost race.
|
|
580
|
-
*
|
|
581
|
-
* **A `summary` goes in the same write that moves the pointer**, so a reader that sees this generation as current
|
|
582
|
-
* sees the count and metadata that describe it, and every attempt sends the same one. It must be the shape the row's
|
|
583
|
-
* keys call for: sealed for an object written with a key, clear for one written without.
|
|
584
|
-
*/
|
|
585
|
-
export declare function publishGenerationKept(registry: IRegistryDriver, key: GenKey, options?: {
|
|
586
|
-
wrappedDeks?: readonly WrappedDek[];
|
|
587
|
-
expectFrom?: number;
|
|
588
|
-
expectToken?: Token;
|
|
589
|
-
/**
|
|
590
|
-
* The row `expectToken` names, as the caller read it. With it, a row whose token has moved but which differs from
|
|
591
|
-
* this one only in its leases (a reader took or released one) does not refuse the publish: it goes on against the
|
|
592
|
-
* fresh row, after a jittered wait and without the caller redoing its work, up to a bound. Any other difference
|
|
593
|
-
* refuses as ever. Without it, a moved token refuses.
|
|
594
|
-
*/
|
|
595
|
-
expectRow?: RegistryRecord | null;
|
|
596
|
-
/**
|
|
597
|
-
* Publish only while the segment still has **no registry row**.
|
|
598
|
-
*
|
|
599
|
-
* The fence for a caller whose decision rests on the row being ABSENT: every load that found no row sets it,
|
|
600
|
-
* guarded or not. `expectFrom` and `expectToken` cannot express it: both compare against a value read from a
|
|
601
|
-
* row, so when there was no row there is nothing to compare and both are simply omitted — leaving the publish a
|
|
602
|
-
* bare forward-only advance that happily lands over whatever appeared in the meantime.
|
|
603
|
-
*
|
|
604
|
-
* Without it, a guarded load into a segment that did not exist yet would judge "no row", pass its empty and
|
|
605
|
-
* `minRetained` bounds vacuously, and land an EMPTY generation over the ids a concurrent writer had just loaded,
|
|
606
|
-
* reporting `published: true`. An unguarded load would move the pointer over another first load's generation or
|
|
607
|
-
* onto a row `setRetention` made, and, where an erasure had deleted its object above that row's pointer, onto an
|
|
608
|
-
* object that is gone. Materialising into a destination that does not exist yet is the ordinary first run of a
|
|
609
|
-
* pipeline, so the `*Into` verbs reach this as often as `loadSegment` does.
|
|
610
|
-
*/
|
|
611
|
-
expectAbsent?: boolean;
|
|
612
|
-
/**
|
|
613
|
-
* The generation's summary (see {@link RegistrySummary}), written by the write that moves the pointer to it. It
|
|
614
|
-
* names `key.generation`. Absent, the row is left with none for this generation.
|
|
615
|
-
*/
|
|
616
|
-
summary?: RegistrySummary;
|
|
617
|
-
/**
|
|
618
|
-
* The segment's row as the caller already read it (`null`: it found none). The first attempt acts on it
|
|
619
|
-
* instead of reading the row again, which is sound because that attempt writes only under the row's own
|
|
620
|
-
* fence: its compare-and-swap carries this row's token, and with no row it creates create-only, so a row
|
|
621
|
-
* that changed or appeared since makes the write lose and the next attempt reads the row afresh. Only the
|
|
622
|
-
* idempotent "already current" answer writes nothing, so a reused row never gives it.
|
|
623
|
-
*/
|
|
624
|
-
row?: RegistryRecord | null;
|
|
625
|
-
/**
|
|
626
|
-
* The object was written without encryption. Its publish is refused, with {@link KeyUnavailableError}, onto a row
|
|
627
|
-
* that carries key material: a reader of that segment opens every generation with its key, and a later
|
|
628
|
-
* `destroySegment` would attest that shredding it made a readable object unreadable. A caller decides its key
|
|
629
|
-
* from the row it read before the write, so this closes the window in which another writer creates the
|
|
630
|
-
* segment encrypted meanwhile.
|
|
631
|
-
*/
|
|
632
|
-
cleartext?: boolean;
|
|
633
|
-
/**
|
|
634
|
-
* Whether the object under `key` is the one the caller wrote, and throws when it cannot tell. Asked only when a
|
|
635
|
-
* registry write failed and the row then names `key.generation`, to tell this publish's write from another
|
|
636
|
-
* writer's at the same number. Every caller that writes the object passes it. Without it, a write that ended
|
|
637
|
-
* without an answer over a row that names `key.generation` cannot be settled: the publish throws
|
|
638
|
-
* {@link TransientError} and the caller keeps its object.
|
|
639
|
-
*/
|
|
640
|
-
holdsOwnObject?: () => Promise<boolean>;
|
|
641
|
-
/**
|
|
642
|
-
* What a publish waits on before it sends a fresh compare-and-swap, after a write that ended without an answer and
|
|
643
|
-
* left the row as it was. Without a `sleep`, such a write throws at once instead of being sent again.
|
|
644
|
-
*/
|
|
645
|
-
clock?: Yielder;
|
|
646
|
-
/** Spreads that wait (full jitter) so concurrent publishers do not retry together. Without it the wait is the bound. */
|
|
647
|
-
rng?: Rng;
|
|
648
|
-
/**
|
|
649
|
-
* Called once for each registry write that ended without an answer, before the row is read back. A caller whose
|
|
650
|
-
* publish then answers `false` can say that it cannot tell whether its own write landed first: the row may have
|
|
651
|
-
* held its generation for a while before another writer moved on.
|
|
652
|
-
*/
|
|
653
|
-
onUnanswered?: () => void;
|
|
654
|
-
/**
|
|
655
|
-
* The clock that judges whether a lease has ended. With it, a publish over a row that holds ended leases drops
|
|
656
|
-
* them in the write that moves the pointer, which rewrites the row anyway. Without it, leases are left as they are.
|
|
657
|
-
*/
|
|
658
|
-
leasesNow?: () => number;
|
|
659
|
-
/**
|
|
660
|
-
* Clear the row's leases in the write that moves the pointer: the caller's collection deletes every generation
|
|
661
|
-
* below the new pointer whatever a lease says, as a subject erasure's does.
|
|
662
|
-
*/
|
|
663
|
-
clearLeases?: boolean;
|
|
664
|
-
/**
|
|
665
|
-
* How many generations below the new pointer the caller keeps. With it, the write that moves the pointer also
|
|
666
|
-
* records the window in the row (`keptGens`: the newest `keep` of the row's own list and the generation that was
|
|
667
|
-
* current), derived from the very row the write is made against, and the result names the generations the window
|
|
668
|
-
* dropped. Without it, or above the most a row records, the row records none, and a collection lists.
|
|
669
|
-
*/
|
|
670
|
-
keep?: number;
|
|
671
|
-
}): Promise<PublishResult>;
|
|
672
|
-
/** What a publish did: whether the pointer is at the generation, and what the write recorded in the row. */
|
|
673
|
-
export interface PublishResult {
|
|
674
|
-
readonly published: boolean;
|
|
675
|
-
/**
|
|
676
|
-
* What the write that moved the pointer recorded: the list, the generations the window dropped, and the new row's
|
|
677
|
-
* token. Absent when no write of this call is known to have landed (the pointer was already there, or the outcome
|
|
678
|
-
* was settled without knowing which write it was), and a collection then lists.
|
|
679
|
-
*/
|
|
680
|
-
readonly kept?: PublishedKept;
|
|
681
|
-
}
|
|
682
|
-
/**
|
|
683
|
-
* {@link publishGenerationKept}'s answer to "did the pointer move", for a caller with no use for what it recorded:
|
|
684
|
-
* a publish with no `keep` leaves the row recording no list.
|
|
685
|
-
*/
|
|
686
|
-
export declare function publishGeneration(registry: IRegistryDriver, key: GenKey, options?: Parameters<typeof publishGenerationKept>[2]): Promise<boolean>;
|
|
687
|
-
/**
|
|
688
|
-
* Whether the object under `key` is the one `fingerprint` names, from one read of its footer: false when it is another
|
|
689
|
-
* object or gone. Unlike {@link holdsObject}, a read that cannot tell throws, for a caller that must not take "could
|
|
690
|
-
* not look" for "not this object".
|
|
691
|
-
*/
|
|
692
|
-
export declare function provesOwnObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
|
|
693
|
-
/** What a bulk-load wrote — the driver's `{ size, sha256 }` plus a summary of the built generation. */
|
|
694
|
-
export interface BulkLoadResult {
|
|
695
|
-
/** Bytes written to the Storage object. */
|
|
696
|
-
readonly size: number;
|
|
697
|
-
/** The driver's content hash of the object. */
|
|
698
|
-
readonly sha256: string;
|
|
699
|
-
/** Distinct non-empty chunks written (≤ 65536). */
|
|
700
|
-
readonly chunkCount: number;
|
|
701
|
-
/**
|
|
702
|
-
* The DEK freshly minted for this generation, wrapped — present only when the segment is encrypted AND this
|
|
703
|
-
* call minted one (an existing segment reuses its key, which is already on the row).
|
|
704
|
-
*
|
|
705
|
-
* Returned so a caller that deliberately does **not** pass `registry` here — writing now and publishing later,
|
|
706
|
-
* which is how a load applies a guard while the old generation is still authoritative — can carry the key onto
|
|
707
|
-
* its own `publishGeneration`. Without it that publish stores no DEK and the generation it makes current is
|
|
708
|
-
* unreadable: the bytes are encrypted with a key nothing recorded.
|
|
709
|
-
*/
|
|
710
|
-
readonly wrappedDeks?: readonly WrappedDek[];
|
|
711
|
-
/** Whether the object was written encrypted: under the segment's key, or under one this call minted. */
|
|
712
|
-
readonly encrypted: boolean;
|
|
713
|
-
/** The written object's fingerprint, which tells it from an object stored later under the same key. */
|
|
714
|
-
readonly fingerprint: string;
|
|
715
|
-
/** Total distinct ids in the generation (post-dedup). */
|
|
716
|
-
readonly cardinality: number;
|
|
717
|
-
/**
|
|
718
|
-
* The summary of what was written, for the write that makes it current: its id count and metadata, sealed under the
|
|
719
|
-
* segment's key when the object was written with one. A caller that defers the publish (`publish: false`) passes it
|
|
720
|
-
* to {@link publishGeneration}.
|
|
721
|
-
*/
|
|
722
|
-
readonly summary: RegistrySummary;
|
|
723
|
-
/**
|
|
724
|
-
* Whether this generation is now the segment's **current** one — `undefined` when no `registry` was wired, so
|
|
725
|
-
* there is no pointer and no publish step to report on.
|
|
726
|
-
*
|
|
727
|
-
* `false` means the object is durable but a **concurrent writer published a higher generation first**, so this
|
|
728
|
-
* one is an orphan that no reader will ever resolve. The write succeeded and the *load* did not: whatever this
|
|
729
|
-
* call was asked to make the segment contain, the segment does not contain. That is not a detail a caller can
|
|
730
|
-
* be left to infer — a `*Into` materialisation that reported the generation it wrote as "the destination's new
|
|
731
|
-
* current generation" on this path would state a plain untruth — so the flag is on the result and the verbs that
|
|
732
|
-
* promise a published generation check it.
|
|
733
|
-
*/
|
|
734
|
-
readonly becameCurrent?: boolean;
|
|
735
|
-
}
|
|
736
|
-
/**
|
|
737
|
-
* **Bulk-load** a whole immutable generation from a flat id source — the batch "seed/rebuild" entry point.
|
|
738
|
-
* Unlike {@link writeCrbmGeneration} (which takes pre-grouped bitmaps), this
|
|
739
|
-
* consumes an arbitrary, **unsorted** stream of u32 ids (sync or async) and routes each into its chunk's
|
|
740
|
-
* Roaring bitmap as it goes. The input is consumed lazily and ids dedup on insert, so memory is bounded by
|
|
741
|
-
* the **built generation** — the in-memory Roaring representation of the *distinct* set (≤ one bitmap per
|
|
742
|
-
* non-empty chunk) — not by the input length: you can stream a billion duplicate-heavy ids and hold only the
|
|
743
|
-
* distinct result. Note this is `O(distinct set)`, **not** window-bounded like {@link SegmentEngine.intersect}:
|
|
744
|
-
* bulk-load holds the whole generation in RAM, which suits batch seed/rebuild jobs but not an
|
|
745
|
-
* unbounded-cardinality stream (a segment larger than RAM needs external-merge / pre-sorted input — later phase).
|
|
746
|
-
*
|
|
747
|
-
* Each id must be an integer in `[0, 2³²)` ({@link splitId} throws {@link ValidationError} otherwise) — the
|
|
748
|
-
* source is consumed lazily, so a bad id aborts mid-stream without writing a partial object (the driver
|
|
749
|
-
* commits only after the callback resolves). An empty source writes a valid empty generation.
|
|
750
|
-
*
|
|
751
|
-
* Writes a fresh full snapshot of a segment. The caller picks the generation number in `key` — `nextGeneration`
|
|
752
|
-
* computes one from the registry and the bucket, as `nextLoadGeneration` does for a load — and re-using an existing
|
|
753
|
-
* generation throws
|
|
754
|
-
* {@link WriteConflictError} (write-once). Without a registry a `StorageChunkSource` serves the **highest**
|
|
755
|
-
* generation present, so a too-high number silently shadows real data; with one, `publishGeneration` decides.
|
|
756
|
-
*/
|
|
757
|
-
export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenKey, ids: Iterable<number> | AsyncIterable<number> | DecodedLoadInput | ChunkLoadInput, options?: {
|
|
758
|
-
registry?: IRegistryDriver;
|
|
759
|
-
keystore?: IKeystore;
|
|
760
|
-
requireEncryption?: boolean;
|
|
761
|
-
audit?: IAuditSink;
|
|
762
|
-
/** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
|
|
763
|
-
codec?: CodecInterface;
|
|
764
|
-
/**
|
|
765
|
-
* Injected clock. Supplying one makes bulk-load **cooperative**: it yields the event loop periodically so a
|
|
766
|
-
* long load does not stall everything else on the process. Without it the load still completes, just
|
|
767
|
-
* without yielding — which is the pre-existing behaviour, kept so this is purely additive.
|
|
768
|
-
*
|
|
769
|
-
* `@cloudbitmaps/roaring` supplies a real clock by default, so flavor users get cooperative behaviour with
|
|
770
|
-
* no wiring. `core/` cannot default it: it is timer-free by lint, which is exactly why waiting goes through
|
|
771
|
-
* this seam rather than `setTimeout`.
|
|
772
|
-
*/
|
|
773
|
-
clock?: Clock;
|
|
774
|
-
/**
|
|
775
|
-
* Write the object but do **not** advance the pointer (default: publish when a `registry` is wired).
|
|
776
|
-
*
|
|
777
|
-
* For a caller that has to inspect what it wrote before deciding whether it should become current — a
|
|
778
|
-
* guarded {@link loadSegment} is the one in-tree case — because the check is only meaningful while the old
|
|
779
|
-
* generation is still authoritative. The `registry` is still required for an encrypted segment and still
|
|
780
|
-
* consulted: it is where an existing segment's DEK lives, and reusing that key is not optional. The wrapped
|
|
781
|
-
* DEK comes back on the result so the deferred publish can store it.
|
|
782
|
-
*
|
|
783
|
-
* A caller that defers the publish owns what it wrote: an unpublished object sits ABOVE `currentGen`, where
|
|
784
|
-
* generation collection deliberately never looks, so nothing reclaims it until a later generation above it is
|
|
785
|
-
* current.
|
|
786
|
-
*/
|
|
787
|
-
publish?: boolean;
|
|
788
|
-
/**
|
|
789
|
-
* The generation's metadata (see {@link GenerationMetadata}), written into the object and carried in the summary.
|
|
790
|
-
* The caller has checked it against the metadata rules; the writer checks it again, and refuses a record that breaks
|
|
791
|
-
* one before a byte is written.
|
|
792
|
-
*/
|
|
793
|
-
metadata?: GenerationMetadata;
|
|
794
|
-
/**
|
|
795
|
-
* The segment's row as the caller already read it (`null`: it found none), for a caller that defers the
|
|
796
|
-
* publish (`publish: false`) and fences it on that same row. A present cleartext row is then used as read,
|
|
797
|
-
* rather than read again: a row that changes in between (a publish, a drop, a purge and re-create) makes the
|
|
798
|
-
* fenced publish lose, so the object is never published on the strength of the stale read. A row read as
|
|
799
|
-
* absent, or one carrying key material, is read again after the ids, as with no row passed: a first load must
|
|
800
|
-
* see a row another writer created meanwhile, and an encrypted segment's key is unwrapped only from a row read
|
|
801
|
-
* after the ids, so a segment shredded while they streamed is refused before its key is used.
|
|
802
|
-
*/
|
|
803
|
-
row?: RegistryRecord | null;
|
|
804
|
-
}): Promise<BulkLoadResult>;
|
|
805
|
-
/**
|
|
806
|
-
* What a load has already started when it calls {@link bulkLoadAhead}: its number, and the segment's data key, each
|
|
807
|
-
* still in flight so their round trips overlap the encoding. Both are joined where the write needs them, and neither
|
|
808
|
-
* may be left to reject unobserved.
|
|
809
|
-
*/
|
|
810
|
-
export interface LoadAhead {
|
|
811
|
-
/** The number the generation takes (see {@link nextLoadGeneration}); joined just before the object is written. */
|
|
812
|
-
readonly generation: PromiseLike<number>;
|
|
813
|
-
/**
|
|
814
|
-
* The key a caller already asked the keystore to unwrap, with the wrappings it asked about. Used only when the
|
|
815
|
-
* row the write reads after the ids carries exactly these wrappings; otherwise the write unwraps for itself.
|
|
816
|
-
*/
|
|
817
|
-
readonly unwrapped?: {
|
|
818
|
-
readonly wrapped: readonly WrappedDek[];
|
|
819
|
-
readonly aead: PromiseLike<Aead>;
|
|
820
|
-
};
|
|
821
|
-
}
|
|
822
|
-
/** What a load with `requireEncryption` meets on a segment that already has a cleartext generation. */
|
|
823
|
-
export declare function cleartextUnderRequiredEncryption(ref: SegmentRef, currentGen: number): ValidationError;
|
|
824
|
-
/**
|
|
825
|
-
* {@link bulkLoadCrbmGeneration} for a load whose number, and perhaps its key, are still being fetched: the same
|
|
826
|
-
* options, with the same meaning, and the same writes. The number is joined only where the object is written.
|
|
827
|
-
*/
|
|
828
|
-
export declare function bulkLoadAhead(driver: IStorageDriver, ref: SegmentRef, ahead: LoadAhead, ids: Parameters<typeof bulkLoadCrbmGeneration>[2], options?: NonNullable<Parameters<typeof bulkLoadCrbmGeneration>[3]>): Promise<BulkLoadResult>;
|
|
829
|
-
/**
|
|
830
|
-
* Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
|
|
831
|
-
* The reader the write paths use to re-read what they wrote, and the erasure rewrite uses to stream the old
|
|
832
|
-
* generation; the engine's read path goes through {@link CrbmStorageChunkSource} instead, which caches these.
|
|
833
|
-
*/
|
|
834
|
-
export declare function openGenerationReader(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, options?: Omit<CrbmReaderOptions, 'crypto'>): Promise<CrbmReader>;
|
|
835
|
-
/**
|
|
836
|
-
* Re-open a freshly written generation and assert it round-trips exactly what was streamed into it: the same
|
|
837
|
-
* per-chunk key set *and* the same total cardinality (on top of the codec's own per-chunk CRC + footer checks), and the
|
|
838
|
-
* metadata it was given. `expected` is the streaming writer's tally (the stream is consumed, so it can't be
|
|
839
|
-
* re-iterated) — the key-set comparison catches a dropped/extra chunk that a cardinality-only check could miss when
|
|
840
|
-
* two errors cancel out. The count and the metadata are what the generation's row summary is about to claim, so the
|
|
841
|
-
* summary is held to the object it describes before it is published.
|
|
842
|
-
* Throws {@link IntegrityError}: the object is on disk but must not be published.
|
|
843
|
-
*/
|
|
844
|
-
export declare function verifyGeneration(storage: IStorageDriver, key: GenKey, expected: {
|
|
845
|
-
readonly chunkKeys: readonly number[];
|
|
846
|
-
readonly cardinality: number;
|
|
847
|
-
readonly metadata?: GenerationMetadata | undefined;
|
|
848
|
-
}, crypto: CrbmCrypto | undefined): Promise<void>;
|