@cloudbitmaps/core 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-TNWTIAON.js → chunk-YSC7NJWV.js} +314 -216
- package/dist/chunk-YSC7NJWV.js.map +7 -0
- package/dist/core/audit.d.ts +56 -3
- package/dist/core/combine-many.d.ts +13 -4
- package/dist/core/consistency.d.ts +10 -4
- package/dist/core/crbm/fingerprint.d.ts +18 -0
- package/dist/core/crbm/format.d.ts +2 -4
- package/dist/core/crbm/frame.d.ts +10 -0
- package/dist/core/crbm/reader.d.ts +1 -3
- package/dist/core/crbm-storage-source.d.ts +249 -96
- package/dist/core/engine.d.ts +22 -14
- package/dist/core/erase-id.d.ts +62 -26
- package/dist/core/generation-gc.d.ts +3 -2
- package/dist/core/keys.d.ts +28 -0
- package/dist/core/leases.d.ts +7 -0
- package/dist/core/load.d.ts +11 -7
- package/dist/core/metrics.d.ts +5 -5
- package/dist/core/pinned-storage-source.d.ts +3 -3
- package/dist/core/pointer-id.d.ts +19 -0
- package/dist/core/ports.d.ts +131 -35
- package/dist/core/reader-defaults.d.ts +14 -3
- package/dist/core/rollback.d.ts +2 -1
- package/dist/core/summary.d.ts +38 -21
- package/dist/core/token.d.ts +13 -8
- package/dist/driver-kit.d.ts +2 -1
- package/dist/driver-kit.js +11 -4
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +1 -1
- package/dist/drivers/_shared/registry.d.ts +26 -17
- package/dist/drivers/localfs/storage.d.ts +7 -6
- package/dist/drivers/memory.d.ts +8 -6
- package/dist/drivers/retry/retrying-chunk-source.d.ts +1 -5
- package/dist/export/index.d.ts +16 -0
- package/dist/index.d.ts +2 -5
- package/dist/index.js +1288 -1080
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
- package/dist/chunk-TNWTIAON.js.map +0 -7
- package/dist/core/cost.d.ts +0 -351
|
@@ -53,7 +53,9 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
|
|
|
53
53
|
* Hard ceiling on how many segments' readers (each holding a fully-parsed `.crbm` index) are cached at once
|
|
54
54
|
* (default 1024) — a memory bound for a long-running server. Past it, the least-recently-used
|
|
55
55
|
* segment's reader is evicted; the next read of an evicted segment re-opens it (one cheap tail GET, since
|
|
56
|
-
* generations are immutable). Raise it for a big cache working set of small segments.
|
|
56
|
+
* generations are immutable). Raise it for a big cache working set of small segments. A source with a timed refresh
|
|
57
|
+
* also keeps up to 8 times this many segments' resolutions apart from their readers, for the TTL: the re-open then
|
|
58
|
+
* reads no row, and comes only when a read needs the reader, for the index or for a chunk.
|
|
57
59
|
*/
|
|
58
60
|
readonly maxOpenSegments?: number;
|
|
59
61
|
/**
|
|
@@ -67,7 +69,8 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
|
|
|
67
69
|
* that region is at most this ceiling divided by {@link maxOpenSegments} (64 KiB by default) — its fair share — so a small generation's
|
|
68
70
|
* chunks are served from memory with no further request, and a bigger one is read by range. Only a source with a timed
|
|
69
71
|
* pointer refresh ({@link currentGenTtlMs} above 0, a registry and a clock) keeps chunk bytes. Lower it for memory-tight deployments with wide segments; a single
|
|
70
|
-
* segment whose index alone exceeds the budget is still cached (it can't be shrunk) but nothing else alongside.
|
|
72
|
+
* segment whose index alone exceeds the budget is still cached (it can't be shrunk) but nothing else alongside. A
|
|
73
|
+
* sixteenth of it bounds the resolutions a source with a timed refresh keeps besides (4 MiB by default).
|
|
71
74
|
*/
|
|
72
75
|
readonly maxOpenIndexBytes?: number;
|
|
73
76
|
}
|
|
@@ -80,6 +83,15 @@ export interface PinnedObject {
|
|
|
80
83
|
readonly version: string;
|
|
81
84
|
readonly fingerprint?: string;
|
|
82
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* What a live version's resolution named, without the object's fingerprint: `<generation>:<pointerId>`, or
|
|
88
|
+
* `<generation>` with no registry. For a caller that compares a live version, from
|
|
89
|
+
* {@link CrbmStorageChunkSource.currentVersion} or a chunk of its `getChunks`, with a row it read. A live version always
|
|
90
|
+
* ends with the suffix, which holds no `#`, so the match starts at its last `#` and a `pointerId` holding `#` of its own
|
|
91
|
+
* is kept whole. Known limit: a string with no suffix, such as a pin's version or one from another source, loses its end
|
|
92
|
+
* when its `pointerId` itself ends in `#<base 36>.<base 36>`; no caller passes one.
|
|
93
|
+
*/
|
|
94
|
+
export declare function rowVersionOf(version: string): string;
|
|
83
95
|
/** What a pin that leases its generation does between reading the row and opening the object. */
|
|
84
96
|
export interface PinLease {
|
|
85
97
|
/** Write the lease against the row the pin read: its token, or `'moved'` when the pointer is no longer at the generation. */
|
|
@@ -94,7 +106,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
94
106
|
* needs a clock). Within the TTL a segment's Storage bytes are treated as an immutable snapshot; when the TTL
|
|
95
107
|
* elapses the next read cheaply re-resolves `currentGen` and, only if it advanced (a load published),
|
|
96
108
|
* opens the new generation — so a long-lived source observes new generations within the TTL. The engine pairs
|
|
97
|
-
* this with a
|
|
109
|
+
* this with a cache keyed by {@link currentVersion}, which names the object a reader opened, so a bump never serves
|
|
110
|
+
* a stale decoded chunk, and neither does a number taken again by another object. Without a clock or a
|
|
98
111
|
* registry there is no timed refresh, and a segment's reader is replaced only when it is evicted, when a read
|
|
99
112
|
* has to fetch from a generation a sweep deleted, or when {@link invalidate} drops it. A segment with no generation yet is not memoized, so
|
|
100
113
|
* it's re-checked until one exists. **Bounded** by a {@link BoundedLru} ({@link CrbmStorageChunkSourceOptions.maxOpenSegments}, default
|
|
@@ -102,6 +115,21 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
102
115
|
* steady-state memory bound; re-opening an evicted segment is one cheap tail GET.
|
|
103
116
|
*/
|
|
104
117
|
private readonly snapshots;
|
|
118
|
+
/**
|
|
119
|
+
* Each segment's resolution, kept apart from its reader, by segment: what a read of its row found, for the TTL from
|
|
120
|
+
* the instant that read was sent, whether or not the reader cache still holds the segment's reader. So letting a reader
|
|
121
|
+
* go moves nothing: the next read builds a snapshot on the resolution, reads no row, and opens the object when it needs
|
|
122
|
+
* the index or a chunk the chunk cache does not hold (a version, a count and a stat from the row's summary need
|
|
123
|
+
* neither). It holds the resolved fields only ({@link Resolved}): the wrapped keys, never an unwrapped one, so a
|
|
124
|
+
* snapshot built on it unwraps the key again through the keystore. It never holds a resolution that found no
|
|
125
|
+
* generation: the next read of such a segment reads the row.
|
|
126
|
+
*
|
|
127
|
+
* Only a source with a timed refresh has one: without it, letting a reader go is one of the few things that ever moves
|
|
128
|
+
* a read on, and keeping the resolution would keep it forever. **Bounded** by count and by bytes, both derived from
|
|
129
|
+
* the reader cache's: {@link RESOLUTIONS_PER_OPEN_SEGMENT} times its count bound, and its byte bound over
|
|
130
|
+
* {@link RESOLUTION_BYTES_DIVISOR}. An entry heavier than the byte bound is kept alone, as the reader cache keeps one.
|
|
131
|
+
*/
|
|
132
|
+
private readonly resolutions;
|
|
105
133
|
/**
|
|
106
134
|
* Pins ({@link heldKey}) whose object has been found replaced: another object is under its generation's key. Kept,
|
|
107
135
|
* and bounded as the readers are, so that a pin's later chunk reads and any reopen fail at once rather than pay for
|
|
@@ -131,6 +159,13 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
131
159
|
* the disagreement again.
|
|
132
160
|
*/
|
|
133
161
|
private readonly distrusted;
|
|
162
|
+
/**
|
|
163
|
+
* The readers held to the row's summary: by a live open ({@link openLive}), or by a live pin that shared one. A pin
|
|
164
|
+
* that finds one memoised under its version shares it as it is: the version names the row's `pointerId`, and every
|
|
165
|
+
* write of the summary renews it, so the summary it was held to is the one the row has now. Any other memoised reader
|
|
166
|
+
* (a `pinAt`'s, held to the fingerprint its caller named) is held to the row's summary before a live pin shares it.
|
|
167
|
+
*/
|
|
168
|
+
private readonly rowChecked;
|
|
134
169
|
private readonly registry;
|
|
135
170
|
private readonly keystore;
|
|
136
171
|
private readonly requireEncryption;
|
|
@@ -141,76 +176,103 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
141
176
|
private readonly currentGenTtlMs;
|
|
142
177
|
/**
|
|
143
178
|
* Whether the segment's pointer is re-read on a timer: it needs a clock (the TTL), a registry (the cheap row read; with
|
|
144
|
-
* none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes
|
|
145
|
-
*
|
|
179
|
+
* none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes and the TTL
|
|
180
|
+
* lapse, so it is decided once.
|
|
146
181
|
*/
|
|
147
182
|
private readonly timedRefresh;
|
|
148
183
|
constructor(driver: IStorageDriver, options?: CrbmStorageChunkSourceOptions);
|
|
149
184
|
/**
|
|
150
|
-
* The
|
|
151
|
-
*
|
|
152
|
-
*
|
|
185
|
+
* The keys of the segments this source reads, kept rather than encoded again: a read asks for its segment's snapshot
|
|
186
|
+
* once per operand, and a stream before every cached chunk it serves, and encoding the names cost about a
|
|
187
|
+
* microsecond each time.
|
|
153
188
|
*/
|
|
154
|
-
private readonly
|
|
189
|
+
private readonly keys;
|
|
155
190
|
private keyOf;
|
|
156
191
|
/**
|
|
157
|
-
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window
|
|
158
|
-
*
|
|
192
|
+
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window: the snapshot the reader cache holds,
|
|
193
|
+
* while its resolution is inside the TTL. With a timed refresh, a segment whose reader the cache let go is served a
|
|
194
|
+
* snapshot built on its kept resolution, while that is inside the TTL: no row is read, and the reader opens only when a
|
|
195
|
+
* read needs one. Otherwise the pointer is read now. The snapshot holds the resolved target, and its reader once a read
|
|
196
|
+
* has asked for one.
|
|
159
197
|
*/
|
|
160
198
|
private liveSnapshot;
|
|
161
|
-
/** The current resolved reader for a segment, refreshing on the TTL. */
|
|
162
|
-
private resolvedReader;
|
|
163
199
|
/**
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
200
|
+
* Read the segment's pointer now. The read is sent before this returns, and the TTL counts from this instant: a read
|
|
201
|
+
* that answers late does not lengthen the life of what it found. The row read is always fresh, so what a snapshot on
|
|
202
|
+
* it says of the generation (its summary included) is what the row says now, never `prior`'s.
|
|
203
|
+
*
|
|
204
|
+
* A read that fails with a transient fault answers what `prior` found, when it found a generation, and is asked again
|
|
205
|
+
* soon ({@link retrySoon}): `prior` is the resolution of the snapshot the reader cache still holds, so an outage of the
|
|
206
|
+
* registry keeps that snapshot's generation, and its reader and unwrapped key where it has them; one resolved without
|
|
207
|
+
* an open, for a count or a generation lookup, unwraps the key from `prior`'s wrapped keys when a read first needs
|
|
208
|
+
* it. The outage ends shortly after the registry answers.
|
|
209
|
+
* That answer is the snapshot's alone ({@link Resolution.rodeOut}): it is not kept as the segment's resolution, and a
|
|
210
|
+
* snapshot built on it after the reader cache let that one go fails with the fault. With no `prior` the fault
|
|
211
|
+
* fails the read, as a cold resolve's does. If `prior` found none or failed, the pointer is read once more rather than
|
|
212
|
+
* a dead resolution re-armed (else the segment reads empty for a whole TTL window). Anything else — an access denial, a corrupt row, a
|
|
213
|
+
* registry that answers NotFound — fails the read exactly as a cold resolve of the segment would, and the resolution
|
|
214
|
+
* and the snapshot on it are forgotten, so a reader and the key it unwrapped do not outlive a refresh that could not
|
|
215
|
+
* be trusted.
|
|
216
|
+
*/
|
|
217
|
+
private resolveNow;
|
|
218
|
+
/**
|
|
219
|
+
* A refresh failed transiently and the prior resolution keeps serving: make it lapse after {@link REFRESH_RETRY_MS}
|
|
220
|
+
* (never longer than the TTL) rather than after a whole TTL, so the next read after that asks the registry again. It
|
|
221
|
+
* costs no read of its own; it only changes when the next refresh happens.
|
|
167
222
|
*/
|
|
168
223
|
private retrySoon;
|
|
169
224
|
/**
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
*
|
|
225
|
+
* Keep `resolution` as the segment's, from the instant its read was sent, so the reads that come while it is in flight
|
|
226
|
+
* share it, and a read after the reader cache let the segment go builds on it. Its weight is reported once it lands,
|
|
227
|
+
* and one that finds no generation, or fails, is forgotten. Both are identity-guarded, as the reader cache's are, and
|
|
228
|
+
* nothing is installed when the read lands: an `invalidate` while it is in flight, or a newer resolution, is never
|
|
229
|
+
* undone by its answer.
|
|
173
230
|
*/
|
|
174
|
-
private
|
|
231
|
+
private remember;
|
|
175
232
|
/**
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
233
|
+
* A live snapshot on `resolution`. `before` is the snapshot it replaces, while the reader cache still holds one: its
|
|
234
|
+
* reader is kept if the resolution finds the same generation, `pointerId` and object ({@link Snapshot.reuse}), and its
|
|
235
|
+
* unwrapped key if the same wrapped keys. A snapshot built after the reader cache let the segment go has neither, so
|
|
236
|
+
* its open asks the keystore again: a key is held only while a reader of the segment is.
|
|
179
237
|
*/
|
|
180
|
-
private
|
|
181
|
-
|
|
238
|
+
private snapshotOn;
|
|
239
|
+
/** What the snapshot on `resolution` that replaces `before` reads: see {@link snapshotOn}. */
|
|
240
|
+
private targetOn;
|
|
182
241
|
/**
|
|
183
|
-
*
|
|
184
|
-
*
|
|
185
|
-
*
|
|
242
|
+
* Memoize a snapshot as the segment's, forgetting it later if it resolves to null / throws (identity-guarded, so a
|
|
243
|
+
* stale cleanup never clobbers a fresher snapshot). Installing the (pending) snapshot **synchronously** is what lets
|
|
244
|
+
* concurrent callers coalesce onto one in-flight resolve.
|
|
186
245
|
*/
|
|
187
|
-
|
|
246
|
+
private install;
|
|
247
|
+
private now;
|
|
188
248
|
/**
|
|
189
249
|
* Whether a stream reading from `snap` must resolve its segment again before its next chunk, as a read of one chunk
|
|
190
|
-
* would
|
|
191
|
-
*
|
|
192
|
-
*
|
|
250
|
+
* would. In this order: an invalidation has happened since `epoch` (what the store that erased stops on, at once);
|
|
251
|
+
* the TTL of the snapshot's resolution has lapsed; or that resolution is no longer the segment's: another read
|
|
252
|
+
* installed a newer one (a refresh, or a heal), or the resolution cache let it go. With no resolution cache (no timed
|
|
253
|
+
* refresh) the last test is of the snapshot itself, so there the reader cache letting it go is a move, as it is one of
|
|
254
|
+
* the few things that ever moves such a store's read on. With one, it is not. Else nothing can have moved it, and the
|
|
255
|
+
* check costs a compare and a lookup in each cache.
|
|
193
256
|
*/
|
|
194
257
|
private mayHaveMoved;
|
|
195
258
|
private expired;
|
|
196
259
|
/**
|
|
197
|
-
* The segment's current generation number
|
|
198
|
-
*
|
|
199
|
-
*
|
|
260
|
+
* The segment's current generation number, as its snapshot's resolution found it: no backend read within the TTL
|
|
261
|
+
* window, and no open of the object, ever. `null` if the segment has no committed generation. A generation swept from
|
|
262
|
+
* under the resolution is met by the read that fetches from it, which heals ({@link withFreshSnapshot}). Since it
|
|
263
|
+
* opens nothing, the only error it meets is the resolution's own, and it fails with that error at once: there is
|
|
264
|
+
* nothing to heal. The reads that can open the object (`listChunkKeys`, `summary`, `currentVersion` and the rest)
|
|
265
|
+
* resolve once more on any `NotFoundError`, a registry's among them, since theirs can be a swept generation.
|
|
200
266
|
*
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
* operand of every `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with
|
|
206
|
-
* no backend call at all. Routing it through the generic helper cost ~115 ns/op on that path (a second async
|
|
207
|
-
* frame, a per-call closure, and an `await` on a plain number) for a race that fires only during a
|
|
208
|
-
* concurrent sweep. One retry, then propagate — same contract, same eviction rule.
|
|
267
|
+
* It is spelled out rather than delegated because a lookup like this runs once per operand of every
|
|
268
|
+
* `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with no backend call at all.
|
|
269
|
+
* Routing it through the generic helper cost ~115 ns/op on that path (a second async frame, a per-call closure, and
|
|
270
|
+
* an `await` on a plain number).
|
|
209
271
|
*/
|
|
210
272
|
currentGeneration(ref: SegmentRef): Promise<number | null>;
|
|
211
273
|
/**
|
|
212
|
-
* Forget everything derived from `ref`: the resolved snapshot and the open reader behind it (and with
|
|
213
|
-
* reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
|
|
274
|
+
* Forget everything derived from `ref`: its resolution, the resolved snapshot and the open reader behind it (and with
|
|
275
|
+
* the reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
|
|
214
276
|
* resolves the segment again.
|
|
215
277
|
*
|
|
216
278
|
* Needed because the TTL refresh only ever answers "has the pointer moved?", and every *destructive* verb —
|
|
@@ -221,21 +283,34 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
221
283
|
*/
|
|
222
284
|
invalidate(ref: SegmentRef): void;
|
|
223
285
|
/**
|
|
224
|
-
* `<generation>` for a registry-less source, or `<generation>:<
|
|
225
|
-
*
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
286
|
+
* `<generation>#<object>` for a registry-less source, or `<generation>:<pointerId>#<object>` with one: the generation
|
|
287
|
+
* the segment's snapshot resolved, and the object a reader of it opens, by that object's size and footer checksum in
|
|
288
|
+
* base 36. The row's `pointerId` moves only with a write that names a field a read resolves through, at any value, so
|
|
289
|
+
* a lease or a policy write (a `setRetention`) leaves the version, the open reader and the decoded chunks as they were.
|
|
290
|
+
* Two objects do not share a version string, as far as their sizes and footer checksums tell them apart, whichever way
|
|
291
|
+
* a number came to name a second one: a name purged and loaded again (a new row, or with no registry the same bare
|
|
292
|
+
* number), a number taken again within one row once its object was deleted, or an object put back under its key from
|
|
293
|
+
* outside the library. The version is opaque: compare two for equality.
|
|
294
|
+
*
|
|
295
|
+
* Where the row's summary names the object, the version is answered from the resolution, with no open: every live
|
|
296
|
+
* open of the generation is held to that object ({@link openLive}), refused as a move otherwise, so the one object any
|
|
297
|
+
* reader of this resolution can name is the one the summary names, and a version named from it is the one that
|
|
298
|
+
* reader's would be. So a `has()` whose chunk is in the chunk cache opens nothing, after the reader cache let
|
|
299
|
+
* the segment go included (a read that needs the index opens the object for it, and on an encrypted segment the
|
|
300
|
+
* key is unwrapped to read the row's sealed summary); an object under the number that is not the row's is met by the read that fetches from it,
|
|
301
|
+
* which refuses it and heals. Where the row has no summary it can use, or there is no registry, nothing names the
|
|
302
|
+
* object but the object: the reader is opened, and the version names what it opened, however long after its row read
|
|
303
|
+
* that is.
|
|
231
304
|
*
|
|
232
|
-
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its
|
|
233
|
-
*
|
|
234
|
-
*
|
|
235
|
-
*
|
|
236
|
-
*
|
|
305
|
+
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its chunk
|
|
306
|
+
* cache, so an open it makes heals a swept generation as a read does: unhealed, a cold read racing a publish and a
|
|
307
|
+
* `keep: 0` sweep failed the whole operation with `NotFoundError`, where the same race on a chunk fetch heals. Spelled
|
|
308
|
+
* out rather than delegated for the reason `currentGeneration` gives: it is almost always served from the cached
|
|
309
|
+
* snapshot with no backend call at all. One retry, then propagate.
|
|
237
310
|
*/
|
|
238
311
|
currentVersion(ref: SegmentRef): Promise<string | null>;
|
|
312
|
+
/** {@link currentVersion} of one snapshot: named by the row's summary when it can be, else by the reader, opened. */
|
|
313
|
+
private versionOn;
|
|
239
314
|
/**
|
|
240
315
|
* Resolve the segment **once** and report what a pin should hold: the generation, and the version that
|
|
241
316
|
* identifies those exact bytes. `null` when the segment resolves to no generation — there is nothing to pin,
|
|
@@ -269,7 +344,7 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
269
344
|
private pinOnce;
|
|
270
345
|
/**
|
|
271
346
|
* Where a pinned reader of one object is memoised. With a registry that is its version, which names the object:
|
|
272
|
-
* a name purged and loaded again gets a new row, and with it a new
|
|
347
|
+
* a name purged and loaded again gets a new row, and with it a new `pointerId`. Without one the version is the bare
|
|
273
348
|
* generation number, which two incarnations share, so the object's fingerprint is added to tell them apart.
|
|
274
349
|
*/
|
|
275
350
|
private pinnedKey;
|
|
@@ -301,9 +376,10 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
301
376
|
* **And the object is checked**, when the caller says which one it pinned. A generation number is not an
|
|
302
377
|
* identity (invariant 1): once a name is purged and loaded again, its new segment starts again at generation 0,
|
|
303
378
|
* so a pin of the old one would open the new one's object as its own, and read it beside the chunks it had
|
|
304
|
-
* already cached from the old one. The row's
|
|
305
|
-
*
|
|
306
|
-
*
|
|
379
|
+
* already cached from the old one. The row's `pointerId` cannot tell them apart, since it moves with every write
|
|
380
|
+
* that changes what the row resolves to, a publish included, while the pinned object stays what it was; the
|
|
381
|
+
* object's fingerprint can, and a pin whose object has been replaced fails, as one whose generation has been swept
|
|
382
|
+
* does. The memo is keyed so that two pins of one generation number in two
|
|
307
383
|
* incarnations never share a reader, with a registry or without one ({@link pinnedKey}).
|
|
308
384
|
*/
|
|
309
385
|
private readerAt;
|
|
@@ -330,7 +406,7 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
330
406
|
* for {@link getChunkAt}.
|
|
331
407
|
*/
|
|
332
408
|
cardinalitiesAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<ReadonlyMap<number, number> | null>;
|
|
333
|
-
/**
|
|
409
|
+
/** Size of a specific generation. `held` as for {@link getChunkAt}. */
|
|
334
410
|
sizeOfAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<SegmentSize | null>;
|
|
335
411
|
/**
|
|
336
412
|
* Whether the segment is a registered name at all. With a registry that is exactly "a row exists" — including
|
|
@@ -343,30 +419,48 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
343
419
|
*/
|
|
344
420
|
exists(ref: SegmentRef): Promise<boolean>;
|
|
345
421
|
/**
|
|
346
|
-
* Evict a snapshot we just failed to read from — matched by
|
|
347
|
-
* fresher snapshot is never clobbered. The identity guard is the whole point: `install`
|
|
348
|
-
* wholesale, so comparing anything else would drop a good
|
|
422
|
+
* Evict a snapshot we just failed to read from, and the resolution it was built on — each matched by identity, so a
|
|
423
|
+
* concurrent call's fresher snapshot or resolution is never clobbered. The identity guard is the whole point: `install`
|
|
424
|
+
* and `remember` replace an entry wholesale, so comparing anything else would drop a good one on the floor. The
|
|
425
|
+
* resolution goes too, or the read's retry would build its snapshot on it again and reopen the generation it just
|
|
426
|
+
* found gone; and so does a snapshot the reader cache holds on that same resolution, which another read built once the
|
|
427
|
+
* cache had let `snap` go, or the retry would open that one.
|
|
349
428
|
*/
|
|
350
429
|
private dropStale;
|
|
351
430
|
/** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
|
|
352
431
|
private openForTarget;
|
|
353
432
|
private openGeneration;
|
|
354
433
|
/**
|
|
355
|
-
* Open the generation a resolution found, and hold the row's summary against the object
|
|
356
|
-
* the
|
|
357
|
-
*
|
|
434
|
+
* Open the generation a resolution found, and hold the row's summary against the object: the object must be the one
|
|
435
|
+
* the summary names by its fingerprint, and is refused before its index is opened otherwise, as another object under
|
|
436
|
+
* the generation's number ({@link NotFoundError}, so the read resolves the row again, as for a swept generation); and
|
|
437
|
+
* it should hold the same count and metadata, or the summary is not believed again in this process
|
|
438
|
+
* ({@link distrusted}). Neither check costs a request. A row with no summary it can use names no object, and its
|
|
439
|
+
* generation's object is opened as it is.
|
|
358
440
|
*/
|
|
359
441
|
private openLive;
|
|
360
442
|
private distrustKey;
|
|
361
443
|
/** What the row says of the generation, unless it is a summary this process has found to disagree with its object. */
|
|
362
444
|
private trustedSummary;
|
|
363
445
|
/**
|
|
364
|
-
*
|
|
365
|
-
* `list`
|
|
366
|
-
*
|
|
367
|
-
|
|
446
|
+
* What the segment resolves to now: the fields of the registry's authoritative record a read resolves through (one
|
|
447
|
+
* strong read), or the highest generation a `list` of the bucket finds when there is no registry (a registry-less
|
|
448
|
+
* source can only read cleartext: there is nowhere a wrapped DEK could live). `null` for no generation.
|
|
449
|
+
*/
|
|
450
|
+
private resolve;
|
|
451
|
+
/**
|
|
452
|
+
* The segment resolved afresh for a pin, with the row it read, which a lease is written against. Never a kept
|
|
453
|
+
* resolution, and kept nowhere: the generation current right now is the whole promise of a pin.
|
|
368
454
|
*/
|
|
369
455
|
private resolveLive;
|
|
456
|
+
/**
|
|
457
|
+
* What a snapshot, or a pin, makes of a resolution: its decryption context and what the row's summary says of the
|
|
458
|
+
* generation, each made only when asked for. A snapshot that replaces `prior` with the same wrapped keys keeps the key
|
|
459
|
+
* `prior` unwrapped, so a count polling an encrypted segment asks the keystore once, not once each `genTtlMs`.
|
|
460
|
+
* Different keys (a shred, a re-created name) unwrap afresh, and so does a snapshot with no prior: the resolution
|
|
461
|
+
* holds the wrapped keys, never an unwrapped one.
|
|
462
|
+
*/
|
|
463
|
+
private liveOf;
|
|
370
464
|
/** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
|
|
371
465
|
private cryptoForRead;
|
|
372
466
|
getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
|
|
@@ -394,16 +488,35 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
394
488
|
* it can: the row's summary of the generation it names, on an active row, in the shape the row's keys call for
|
|
395
489
|
* (a sealed one is opened with the segment's key), and one this process has not found to disagree with its
|
|
396
490
|
* object. That is one registry read when the snapshot is cold, none when it is warm, and no read of the object.
|
|
397
|
-
* With no summary it can use (
|
|
398
|
-
* generation and answers from the object. A row that names an object that is gone
|
|
399
|
-
* of the object throws: the number is true of the generation the row
|
|
491
|
+
* With no summary it can use (a row written without one, one for another generation, one that does not open), it
|
|
492
|
+
* opens the generation and answers from the object. A row that names an object that is gone, or another object
|
|
493
|
+
* under the number, still answers, while a read of the object throws: the number is true of the generation the row
|
|
494
|
+
* names.
|
|
400
495
|
*
|
|
401
496
|
* What the row says is not confirmed here: a summary edited to agree with itself by whoever can write the row is
|
|
402
497
|
* believed until the next time the object is opened, which holds it against the summary.
|
|
403
498
|
*/
|
|
404
499
|
summary(ref: SegmentRef): Promise<GenerationSummary | null>;
|
|
500
|
+
/**
|
|
501
|
+
* What one resolution of the segment says of its generation: from the row's summary when this process may use it,
|
|
502
|
+
* else from the generation's object, opened. `null` for no generation. One heal, as a read of the object has: a
|
|
503
|
+
* generation swept from under the snapshot, or replaced under its number, is resolved again once.
|
|
504
|
+
*/
|
|
505
|
+
private described;
|
|
405
506
|
/** {@link summary} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
406
507
|
summaryAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<GenerationSummary | null>;
|
|
508
|
+
/**
|
|
509
|
+
* The current generation's number, id count and metadata, with its object's size, all from one resolution. The answer
|
|
510
|
+
* comes from the row when it can, as {@link summary}'s does: the row's summary of the generation it names, whose
|
|
511
|
+
* fingerprint carries the object's size. That is one registry read when the snapshot is cold, none when it is warm,
|
|
512
|
+
* and no read of the object; on an encrypted segment the key is unwrapped to open the summary, as for a count. With no
|
|
513
|
+
* summary it can use, it opens the generation and answers from its footer and index, with no payload read: a tail read
|
|
514
|
+
* more when cold. A row that names an object that is gone, or another object, still answers from the row: the figures
|
|
515
|
+
* are the generation's the row names, and `checkConsistency` is what finds such a row.
|
|
516
|
+
*/
|
|
517
|
+
stat(ref: SegmentRef): Promise<(GenerationSummary & SegmentSize) | null>;
|
|
518
|
+
/** {@link stat} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
519
|
+
statAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<(GenerationSummary & SegmentSize) | null>;
|
|
407
520
|
/**
|
|
408
521
|
* Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
|
|
409
522
|
* generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
|
|
@@ -411,8 +524,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
411
524
|
* caller-visible error, so instead we drop the stale snapshot,
|
|
412
525
|
* re-resolve `currentGen`, and retry once — the read then serves the newer (committed, immutable) generation,
|
|
413
526
|
* a monotonic move forward — within one incarnation of the row. A name that was retired and re-created is a
|
|
414
|
-
* different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's
|
|
415
|
-
* as its lineage rather than trusting the generation number (
|
|
527
|
+
* different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's
|
|
528
|
+
* `pointerId` as its lineage rather than trusting the generation number (invariants 1 and 2).
|
|
416
529
|
*
|
|
417
530
|
* **Resolution and open are inside the retry, not before it.** Resolving `currentGen` and opening that
|
|
418
531
|
* generation's object are two backend round trips with a gap between them, so the object can be swept after
|
|
@@ -462,13 +575,27 @@ export declare function writeCrbmGeneration(driver: IStorageDriver, key: GenKey,
|
|
|
462
575
|
fingerprint: string;
|
|
463
576
|
}>;
|
|
464
577
|
/**
|
|
465
|
-
*
|
|
466
|
-
* read of its footer
|
|
467
|
-
*
|
|
578
|
+
* The object under `key`, when it is provably the one `fingerprint` names ({@link CrbmReader.fingerprint}), from one
|
|
579
|
+
* read of its footer, with the version the driver reported on that read (`undefined` from a driver that reports none).
|
|
580
|
+
* `null` when it is another object, when it is gone, and when the read fails in any way: a caller that deletes what
|
|
581
|
+
* this proves deletes nothing it cannot prove is its own, and passes the version as the delete's `ifVersion`, so a
|
|
582
|
+
* number taken again between the read and the delete keeps the object put under it since.
|
|
583
|
+
*/
|
|
584
|
+
export declare function provenObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<{
|
|
585
|
+
readonly version: string | undefined;
|
|
586
|
+
} | null>;
|
|
587
|
+
/**
|
|
588
|
+
* Whether the object under `key` is still the one `fingerprint` names ({@link CrbmReader.fingerprint}), from one read
|
|
589
|
+
* of its footer. Unlike {@link provenObject}, a fault is not an answer: `NotFoundError` when the key holds no object,
|
|
590
|
+
* and any other failure of the read, thrown as it is.
|
|
591
|
+
*/
|
|
592
|
+
export declare function sameObjectUnder(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
|
|
593
|
+
/**
|
|
594
|
+
* Whether the object under `key` says it is encrypted, from one read of its footer, with no key. `noteVersion` is shown
|
|
595
|
+
* the version the driver reported on that read, when it reported one: a caller that deletes the object for what its
|
|
596
|
+
* footer says passes it as the delete's `ifVersion`.
|
|
468
597
|
*/
|
|
469
|
-
export declare function
|
|
470
|
-
/** Whether the object under `key` says it is encrypted, from one read of its footer, with no key. */
|
|
471
|
-
export declare function objectIsEncrypted(storage: IStorageDriver, key: GenKey): Promise<boolean>;
|
|
598
|
+
export declare function objectIsEncrypted(storage: IStorageDriver, key: GenKey, noteVersion?: (version: string) => void): Promise<boolean>;
|
|
472
599
|
/**
|
|
473
600
|
* What one tail read of the object under `key` tells a rollback about it: whether the footer says it is encrypted, and,
|
|
474
601
|
* when the object and the key agree (an encrypted object opened with `crypto`, a cleartext one with none), the reader
|
|
@@ -583,17 +710,17 @@ export declare function publishGenerationKept(registry: IRegistryDriver, key: Ge
|
|
|
583
710
|
/**
|
|
584
711
|
* Publish only while the segment still has **no registry row**.
|
|
585
712
|
*
|
|
586
|
-
* The fence for a caller whose decision rests on the row being ABSENT
|
|
587
|
-
* cannot express it: both compare against a value read from a
|
|
588
|
-
* nothing to compare and both are simply omitted — leaving the publish a
|
|
589
|
-
* happily lands over whatever appeared in the meantime.
|
|
713
|
+
* The fence for a caller whose decision rests on the row being ABSENT: every load that found no row sets it,
|
|
714
|
+
* guarded or not. `expectFrom` and `expectToken` cannot express it: both compare against a value read from a
|
|
715
|
+
* row, so when there was no row there is nothing to compare and both are simply omitted — leaving the publish a
|
|
716
|
+
* bare forward-only advance that happily lands over whatever appeared in the meantime.
|
|
590
717
|
*
|
|
591
|
-
*
|
|
592
|
-
*
|
|
593
|
-
*
|
|
594
|
-
*
|
|
595
|
-
*
|
|
596
|
-
*
|
|
718
|
+
* Without it, a guarded load into a segment that did not exist yet would judge "no row", pass its empty and
|
|
719
|
+
* `minRetained` bounds vacuously, and land an EMPTY generation over the ids a concurrent writer had just loaded,
|
|
720
|
+
* reporting `published: true`. An unguarded load would move the pointer over another first load's generation or
|
|
721
|
+
* onto a row `setRetention` made, and, where an erasure had deleted its object above that row's pointer, onto an
|
|
722
|
+
* object that is gone. Materialising into a destination that does not exist yet is the ordinary first run of a
|
|
723
|
+
* pipeline, so the `*Into` verbs reach this as often as `loadSegment` does.
|
|
597
724
|
*/
|
|
598
725
|
expectAbsent?: boolean;
|
|
599
726
|
/**
|
|
@@ -665,6 +792,12 @@ export interface PublishResult {
|
|
|
665
792
|
* was settled without knowing which write it was), and a collection then lists.
|
|
666
793
|
*/
|
|
667
794
|
readonly kept?: PublishedKept;
|
|
795
|
+
/**
|
|
796
|
+
* A token of the row whose pointer is at the generation, when it is published: the one the write was given, or the
|
|
797
|
+
* row read that showed the pointer there. Every write of a row keeps its incarnation, so this names the incarnation
|
|
798
|
+
* the publish landed in, for the audit event.
|
|
799
|
+
*/
|
|
800
|
+
readonly token?: Token;
|
|
668
801
|
}
|
|
669
802
|
/**
|
|
670
803
|
* {@link publishGenerationKept}'s answer to "did the pointer move", for a caller with no use for what it recorded:
|
|
@@ -673,7 +806,7 @@ export interface PublishResult {
|
|
|
673
806
|
export declare function publishGeneration(registry: IRegistryDriver, key: GenKey, options?: Parameters<typeof publishGenerationKept>[2]): Promise<boolean>;
|
|
674
807
|
/**
|
|
675
808
|
* Whether the object under `key` is the one `fingerprint` names, from one read of its footer: false when it is another
|
|
676
|
-
* object or gone. Unlike {@link
|
|
809
|
+
* object or gone. Unlike {@link provenObject}, a read that cannot tell throws, for a caller that must not take "could
|
|
677
810
|
* not look" for "not this object".
|
|
678
811
|
*/
|
|
679
812
|
export declare function provesOwnObject(storage: IStorageDriver, key: GenKey, fingerprint: string): Promise<boolean>;
|
|
@@ -816,9 +949,29 @@ export declare function bulkLoadAhead(driver: IStorageDriver, ref: SegmentRef, a
|
|
|
816
949
|
/**
|
|
817
950
|
* Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
|
|
818
951
|
* The reader the write paths use to re-read what they wrote, and the erasure rewrite uses to stream the old
|
|
819
|
-
* generation; the engine's read path goes through {@link CrbmStorageChunkSource} instead, which caches these.
|
|
952
|
+
* generation; the engine's read path goes through {@link CrbmStorageChunkSource} instead, which caches these. With
|
|
953
|
+
* `named`, the fingerprint the row's summary records, the object must be that one: another is {@link NotFoundError}.
|
|
954
|
+
* The version of the object it opened, from the tail read the open made, is what {@link objectVersionOf} answers.
|
|
955
|
+
*/
|
|
956
|
+
export declare function openGenerationReader(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, options?: Omit<CrbmReaderOptions, 'crypto'>, named?: string): Promise<CrbmReader>;
|
|
957
|
+
/**
|
|
958
|
+
* {@link openGenerationReader}, with `noteVersion` shown the version {@link objectVersionOf} would answer as soon as the
|
|
959
|
+
* tail read that carries it returns, before the open goes on: a caller whose open then fails, on an object sealed under
|
|
960
|
+
* another key, say, still knows which object it met, and can condition a delete of it on that read.
|
|
961
|
+
*/
|
|
962
|
+
export declare function openGenerationReaderNoting(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, noteVersion: (version: string) => void): Promise<CrbmReader>;
|
|
963
|
+
/**
|
|
964
|
+
* The version of the object `reader` was opened on, as the driver reported it on the tail read the open made, for a
|
|
965
|
+
* delete conditioned on that object ({@link StorageDeleteOptions.ifVersion}); `undefined` for a reader
|
|
966
|
+
* {@link openGenerationReader} did not open, or a driver that reports none. Held beside the reader rather than on it, so
|
|
967
|
+
* nothing is added to the reader's public surface, and no read the engine makes pays for it.
|
|
968
|
+
*/
|
|
969
|
+
export declare function objectVersionOf(reader: CrbmReader): string | undefined;
|
|
970
|
+
/**
|
|
971
|
+
* {@link openGenerationReader}, of the object `fingerprint` names: `undefined` when the object under the key is another
|
|
972
|
+
* one, found from its footer before its index is opened, so an object sealed under another key is told apart as well.
|
|
820
973
|
*/
|
|
821
|
-
export declare function
|
|
974
|
+
export declare function openNamedGeneration(storage: IStorageDriver, key: GenKey, crypto: CrbmCrypto | undefined, fingerprint: string): Promise<CrbmReader | undefined>;
|
|
822
975
|
/**
|
|
823
976
|
* Re-open a freshly written generation and assert it round-trips exactly what was streamed into it: the same
|
|
824
977
|
* per-chunk key set *and* the same total cardinality (on top of the codec's own per-chunk CRC + footer checks), and the
|