@cloudbitmaps/core 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-5OMC5U7U.js → chunk-5W3XS2IU.js} +292 -22
- package/dist/chunk-5W3XS2IU.js.map +7 -0
- package/dist/core/audit.d.ts +6 -0
- package/dist/core/codec.d.ts +33 -1
- package/dist/core/consistency.d.ts +21 -2
- package/dist/core/cost.d.ts +91 -10
- package/dist/core/crbm/format.d.ts +31 -1
- package/dist/core/crbm/reader.d.ts +44 -2
- package/dist/core/crbm/writer.d.ts +32 -3
- package/dist/core/crbm-storage-source.d.ts +203 -22
- package/dist/core/crypto.d.ts +11 -7
- package/dist/core/determinism.d.ts +7 -0
- package/dist/core/engine.d.ts +17 -5
- package/dist/core/erase-id.d.ts +20 -0
- package/dist/core/erasure.d.ts +8 -4
- package/dist/core/generation-gc.d.ts +102 -4
- package/dist/core/load-input.d.ts +55 -0
- package/dist/core/load.d.ts +65 -10
- package/dist/core/metadata.d.ts +42 -0
- package/dist/core/pinned-storage-source.d.ts +2 -1
- package/dist/core/ports.d.ts +119 -13
- package/dist/core/registry-scan.d.ts +6 -0
- package/dist/core/retention-sweep.d.ts +47 -2
- package/dist/core/retry.d.ts +14 -0
- package/dist/core/rollback.d.ts +34 -4
- package/dist/core/summary.d.ts +61 -0
- package/dist/core/token.d.ts +36 -0
- package/dist/core/validate.d.ts +2 -0
- package/dist/driver-kit.js +39 -20
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/entropy.d.ts +5 -0
- package/dist/drivers/_shared/object-registry.d.ts +36 -6
- package/dist/drivers/_shared/registry.d.ts +47 -19
- package/dist/drivers/localfs/registry.d.ts +10 -0
- package/dist/drivers/memory.d.ts +9 -3
- package/dist/drivers/retry/retrying-chunk-source.d.ts +2 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.js +1794 -461
- package/dist/index.js.map +4 -4
- package/dist/testing/fuzz-core.d.ts +6 -5
- package/package.json +1 -1
- package/dist/chunk-5OMC5U7U.js.map +0 -7
package/dist/core/ports.d.ts
CHANGED
|
@@ -59,10 +59,18 @@ export interface StorageChunkSource {
|
|
|
59
59
|
/**
|
|
60
60
|
* Optional: the current generation's **per-chunk cardinality** (`chunkKey → count`), read from the
|
|
61
61
|
* already-parsed `.crbm` index with **no payload reads**, or `null` if the segment has no Storage generation.
|
|
62
|
-
* Powers the free `count()`
|
|
62
|
+
* Powers the free `count()` of a source with no {@link StorageChunkSource.summary} — the engine sums the index
|
|
63
|
+
* instead of fetching a single chunk. A source with no
|
|
63
64
|
* index (e.g. the in-memory source) omits this, and `count()` falls back to fetching every chunk.
|
|
64
65
|
*/
|
|
65
66
|
cardinalities?(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
|
|
67
|
+
/**
|
|
68
|
+
* Optional: the current generation's number, id count and metadata, or `null` if the segment has no Storage
|
|
69
|
+
* generation. The `.crbm` source answers from the registry row's summary when it can use it, which is one
|
|
70
|
+
* registry read and no read of the object, and from the opened generation when it cannot. A source with no
|
|
71
|
+
* summary omits this, and `count()` keeps its other paths.
|
|
72
|
+
*/
|
|
73
|
+
summary?(ref: SegmentRef): Promise<GenerationSummary | null>;
|
|
66
74
|
/**
|
|
67
75
|
* Optional: the segment's **current generation number** as this source resolves it right now (registry
|
|
68
76
|
* `currentGen`, or the highest storage generation), or `null` if the segment has no Storage generation. The engine
|
|
@@ -105,10 +113,10 @@ export interface StorageChunkSource {
|
|
|
105
113
|
* the generation *and* the incarnation of the name it belongs to.
|
|
106
114
|
*
|
|
107
115
|
* {@link StorageChunkSource.currentGeneration} is not enough to key a decoded-chunk cache, and the gap is not
|
|
108
|
-
* theoretical: a load numbers its generation
|
|
109
|
-
* a registry row is purged and the bucket emptied. A retired, re-created name therefore
|
|
110
|
-
* data at the same `currentGen`, and a cache keyed on `(segment, chunk, generation)` hands back
|
|
111
|
-
* incarnation's ids — an erased id reappearing, with no read to intercept.
|
|
116
|
+
* theoretical: a load numbers its generation from the row's pointer and what is in the bucket, so the number
|
|
117
|
+
* **restarts at 0** once a registry row is purged and the bucket emptied. A retired, re-created name therefore
|
|
118
|
+
* serves different data at the same `currentGen`, and a cache keyed on `(segment, chunk, generation)` hands back
|
|
119
|
+
* the previous incarnation's ids — an erased id reappearing, with no read to intercept.
|
|
112
120
|
*
|
|
113
121
|
* Note that putting the incarnation in the *object key* would not fix this: the new incarnation still starts
|
|
114
122
|
* at generation 0, so the cache key collides either way. The identity has to reach the cache.
|
|
@@ -160,8 +168,9 @@ export interface StorageCaps {
|
|
|
160
168
|
* loses its response, sent again, finds its own object and would report a collision. Either send each write
|
|
161
169
|
* once, with the backend client's retry off for that request, or recognise your own write when the
|
|
162
170
|
* precondition fails by reading an id you stored with it back, and treat a match as success. The first is
|
|
163
|
-
* what the
|
|
164
|
-
* resumable GCS upload do
|
|
171
|
+
* what the registry writes of every shipped driver do; the second is what the Azure Blob driver and the
|
|
172
|
+
* resumable GCS upload do, and what the S3 driver and the single-request GCS upload do when they send an object
|
|
173
|
+
* again after a throttle, which is the one replay a driver may make on purpose.
|
|
165
174
|
* - **Raise a transient fault as {@link TransientError}** (throttling, a 5xx, a dropped connection), so the read
|
|
166
175
|
* retry can ride it out and a write's caller can tell it from a deterministic failure.
|
|
167
176
|
*/
|
|
@@ -212,6 +221,50 @@ export type RegistryStatus = 'active' | 'destroyed';
|
|
|
212
221
|
* shape-later: stored and round-tripped, no semantics attached.
|
|
213
222
|
*/
|
|
214
223
|
export type GovernanceMeta = Record<string, unknown>;
|
|
224
|
+
/**
|
|
225
|
+
* A cached description of one generation, carried on the registry row: how many ids it holds, and the metadata it
|
|
226
|
+
* was loaded with. The generation's `.crbm` object stays the truth; this is a copy, written by the same write that
|
|
227
|
+
* moves the pointer, and it names the generation it describes so a copy left behind by a writer that did not
|
|
228
|
+
* carry it can never be taken for another generation's.
|
|
229
|
+
*
|
|
230
|
+
* Two shapes. On a cleartext segment the values are in the clear. On an encrypted segment (a row with
|
|
231
|
+
* `wrappedDeks`) they are sealed under the segment's data key, as the generation's index is, so the row reveals
|
|
232
|
+
* neither the count nor the metadata. A shipped registry refuses a write whose summary disagrees with the row's
|
|
233
|
+
* keys. It still reads a stored row that disagrees, so that one such row cannot stop every listing: a reader must not
|
|
234
|
+
* use that row's summary.
|
|
235
|
+
*/
|
|
236
|
+
export type RegistrySummary = ClearRegistrySummary | SealedRegistrySummary;
|
|
237
|
+
/**
|
|
238
|
+
* A generation's metadata: string keys, string or finite-number values, no nesting, at most 1 KiB as canonical JSON.
|
|
239
|
+
* The rules are in `core/metadata.ts`.
|
|
240
|
+
*/
|
|
241
|
+
export type GenerationMetadata = Readonly<Record<string, string | number>>;
|
|
242
|
+
/** What a generation is, as one read answers it: its number, its id count and the metadata it was written with. */
|
|
243
|
+
export interface GenerationSummary {
|
|
244
|
+
readonly generation: number;
|
|
245
|
+
readonly cardinality: number;
|
|
246
|
+
/** Absent when the generation has none. */
|
|
247
|
+
readonly metadata?: GenerationMetadata;
|
|
248
|
+
}
|
|
249
|
+
/** {@link RegistrySummary} on a cleartext segment. */
|
|
250
|
+
export interface ClearRegistrySummary {
|
|
251
|
+
/** The generation this describes. A non-negative safe integer. */
|
|
252
|
+
readonly generation: number;
|
|
253
|
+
/** How many ids the generation holds: an integer from 0 to 2^32. */
|
|
254
|
+
readonly cardinality: number;
|
|
255
|
+
/** The generation's metadata, when it has any. Never the empty object. */
|
|
256
|
+
readonly metadata?: GenerationMetadata;
|
|
257
|
+
}
|
|
258
|
+
/** {@link RegistrySummary} on an encrypted segment. */
|
|
259
|
+
export interface SealedRegistrySummary {
|
|
260
|
+
/** The generation this describes. A non-negative safe integer. */
|
|
261
|
+
readonly generation: number;
|
|
262
|
+
/**
|
|
263
|
+
* Base64 of `nonce(12) ‖ ciphertext ‖ tag(16)`, sealing the cardinality as a little-endian u64 followed by the
|
|
264
|
+
* metadata's canonical JSON, if any. The count is fixed-width, so the length reveals only the metadata's size.
|
|
265
|
+
*/
|
|
266
|
+
readonly sealed: string;
|
|
267
|
+
}
|
|
215
268
|
/**
|
|
216
269
|
* One registry row — the authoritative per-segment record. Exactly one per segment.
|
|
217
270
|
*/
|
|
@@ -249,10 +302,23 @@ export interface RegistryRecord extends SegmentRef {
|
|
|
249
302
|
*/
|
|
250
303
|
readonly retention?: GovernanceMeta;
|
|
251
304
|
readonly residency?: GovernanceMeta;
|
|
305
|
+
/**
|
|
306
|
+
* A cached description of the current generation (see {@link RegistrySummary}). Optional: a row without one is
|
|
307
|
+
* correct, and a reader then opens the generation instead. Trusted only for the generation it names.
|
|
308
|
+
*/
|
|
309
|
+
readonly summary?: RegistrySummary;
|
|
252
310
|
/** Epoch-ms of creation / last mutation (from the driver's injected clock). */
|
|
253
311
|
readonly createdAt: number;
|
|
254
312
|
readonly updatedAt: number;
|
|
255
|
-
/**
|
|
313
|
+
/**
|
|
314
|
+
* Opaque OCC token — compare-by-equality, ABA-safe. A shipped registry gives no two writes under one name the same
|
|
315
|
+
* token: not two writes of this row, not a write of an earlier row under the name, and not a write after this row is
|
|
316
|
+
* restored from a backup. It holds with overwhelming probability rather than by construction, since the token carries random parts: a
|
|
317
|
+
* 128-bit incarnation id drawn when the row is created, and a 64-bit part drawn for each write. Two incarnations of one
|
|
318
|
+
* name meet with probability 2^-128 for any pair (about n^2 / 2^129 among n of them), and two writes at one counter,
|
|
319
|
+
* after a restore, with probability 2^-64. A bare decimal token, on a row no 0.12 or later registry has written,
|
|
320
|
+
* carries neither part.
|
|
321
|
+
*/
|
|
256
322
|
readonly token: Token;
|
|
257
323
|
}
|
|
258
324
|
/** The caller-settable fields at {@link IRegistryDriver.create} (audit + token are driver-managed). */
|
|
@@ -265,14 +331,47 @@ export interface NewRegistryRecord {
|
|
|
265
331
|
readonly status?: RegistryStatus;
|
|
266
332
|
readonly retention?: GovernanceMeta;
|
|
267
333
|
readonly residency?: GovernanceMeta;
|
|
334
|
+
/** Must name `currentGen` when given. */
|
|
335
|
+
readonly summary?: RegistrySummary;
|
|
268
336
|
}
|
|
269
|
-
/**
|
|
270
|
-
|
|
337
|
+
/**
|
|
338
|
+
* Fields a {@link IRegistryDriver.compareAndSwap} may mutate (identity + audit + token are off-limits).
|
|
339
|
+
*
|
|
340
|
+
* Presence-based: a field the patch does not mention is left as it was. `summary` is the one exception, because it
|
|
341
|
+
* describes the current generation: a patch that moves `currentGen` and does not mention `summary` drops the old
|
|
342
|
+
* one rather than keep a description of another generation. A `summary` the patch gives must name the
|
|
343
|
+
* `currentGen` the row will have.
|
|
344
|
+
*/
|
|
345
|
+
export type RegistryPatch = Partial<Pick<RegistryRecord, 'currentGen' | 'wrappedDeks' | 'keyId' | 'status' | 'retention' | 'residency' | 'summary'>>;
|
|
271
346
|
/** Capabilities a registry driver advertises; validated fail-fast at wiring time. */
|
|
272
347
|
export interface RegCaps {
|
|
273
348
|
/** REQUIRED — `currentGen` feeds read correctness + the publish CAS, so reads must be strongly consistent. */
|
|
274
349
|
readonly strongRead: true;
|
|
350
|
+
/**
|
|
351
|
+
* `false` when this registry cannot write a row in this runtime, though it reads: a shipped registry draws random
|
|
352
|
+
* bytes for every token it issues, and on a runtime with no Web Crypto it has none to draw. A write path that
|
|
353
|
+
* writes an object before its row (a load, an erasure rewrite) checks it first and refuses with `UnsupportedError`
|
|
354
|
+
* before it writes anything, rather than leave an object no row names. Absent means the registry can write.
|
|
355
|
+
*/
|
|
356
|
+
readonly canWrite?: false;
|
|
357
|
+
/**
|
|
358
|
+
* `true` when this registry's `delete` removes a row from its backend for good, so that no later `list` reads it, and
|
|
359
|
+
* removes it only while it is still the exact version the delete read: by a delete the backend applies under a
|
|
360
|
+
* precondition (an S3 or Azure Blob ETag, a GCS object generation), or under a lock no other writer of the backend
|
|
361
|
+
* can take. Only a row whose token carries an incarnation id is removed so. A row first written by a release before
|
|
362
|
+
* 0.12 has a bare decimal token, and a delete still tombstones it: a process on that release, re-creating the name
|
|
363
|
+
* over nothing, would issue those counters again from 0, so its row could not be told apart from the deleted one.
|
|
364
|
+
*
|
|
365
|
+
* `false` or absent: every `delete` leaves a tombstone, which every later full `list` still reads. A shipped registry
|
|
366
|
+
* says which it is.
|
|
367
|
+
*/
|
|
368
|
+
readonly conditionalDelete?: boolean;
|
|
275
369
|
}
|
|
370
|
+
/**
|
|
371
|
+
* Refuse, with `UnsupportedError`, a write that would write an object before its row when the registry reports it
|
|
372
|
+
* cannot write a row (see {@link RegCaps.canWrite}). Called before the write's first request.
|
|
373
|
+
*/
|
|
374
|
+
export declare function assertRegistryCanWrite(registry: IRegistryDriver, what: string): void;
|
|
276
375
|
/**
|
|
277
376
|
* Brand for {@link StorageBackend}.
|
|
278
377
|
*
|
|
@@ -365,8 +464,13 @@ export declare function isStorageBackend(value: unknown): value is StorageBacken
|
|
|
365
464
|
* - **`create` and `compareAndSwap` are atomic conditional writes.** A `create` over a live row, and a
|
|
366
465
|
* `compareAndSwap` whose token is not the stored one, throw {@link WriteConflictError} and change nothing.
|
|
367
466
|
* Two racing writers have exactly one winner.
|
|
368
|
-
* - **
|
|
369
|
-
*
|
|
467
|
+
* - **A token is not reused, not even across `delete` then `create`**, short of a collision of probability 2^-128 for
|
|
468
|
+
* any two incarnations of one name: a recreated row carries a token no earlier incarnation held, so a stale holder
|
|
469
|
+
* cannot swap into it. The shipped drivers draw a random 128-bit incarnation id into the token of every row they
|
|
470
|
+
* create and a random 64-bit part into the token of every write, beside a counter a tombstone (or a global counter)
|
|
471
|
+
* keeps going; where nothing of the earlier row is left, or the row is restored from a backup to an older counter, the
|
|
472
|
+
* random parts alone keep the tokens apart: with probability 1 - 2^-128 per pair of incarnations, and 1 - 2^-64 per
|
|
473
|
+
* pair of writes at one counter.
|
|
370
474
|
* - **`delete` is idempotent** without an `expected` token: deleting an absent row is a no-op, not an error.
|
|
371
475
|
* **With `expected` it is fenced**: it lands only while the row still carries that token, and otherwise throws
|
|
372
476
|
* {@link WriteConflictError} and leaves the row.
|
|
@@ -408,7 +512,9 @@ export interface IRegistryDriver {
|
|
|
408
512
|
*/
|
|
409
513
|
list(namespace?: string): AsyncIterable<RegistryRecord>;
|
|
410
514
|
/**
|
|
411
|
-
* Remove the row
|
|
515
|
+
* Remove the row. A registry whose {@link RegCaps.conditionalDelete} is `true` removes a row whose token carries an
|
|
516
|
+
* incarnation id from its backend; every other row is tombstoned. Either way a later `create` gets a token no
|
|
517
|
+
* earlier incarnation of the name held, but for a collision of probability 2^-128 per pair of incarnations.
|
|
412
518
|
*
|
|
413
519
|
* Without `expected`, idempotent: deleting an absent row is a no-op.
|
|
414
520
|
*
|
|
@@ -27,6 +27,12 @@ export declare function drainRegistry(registry: IRegistryDriver, options: {
|
|
|
27
27
|
* one namespace, so it passes false and the refusal does not tell it to narrow what it cannot.
|
|
28
28
|
*/
|
|
29
29
|
narrowable?: boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Told of each bookkeeping row an unscoped scan skips, which the listing has already paid to read. Only the
|
|
32
|
+
* retention sweep uses it, to find the due-index pointers whose segment has no row. It is not counted against
|
|
33
|
+
* `maxScanSegments`, and a caller that holds the rows it is given bounds them itself.
|
|
34
|
+
*/
|
|
35
|
+
onReserved?: (record: RegistryRecord) => void;
|
|
30
36
|
}): Promise<RegistryRecord[]>;
|
|
31
37
|
/**
|
|
32
38
|
* Is this record **bookkeeping** rather than a segment? Covers every reserved family — today the due-index
|
|
@@ -33,11 +33,29 @@
|
|
|
33
33
|
* once Storage is provably empty for it —
|
|
34
34
|
* because deleting the row is what makes the name reusable and takes the segment out of reach of
|
|
35
35
|
* the generation collection.
|
|
36
|
+
*
|
|
37
|
+
* **On a registry that reports `conditionalDelete`, the purge removes the row for good**, so a full scan stops paying a
|
|
38
|
+
* read for every name a namespace ever held: the scan costs what is live and what is inside its grace. The delete is
|
|
39
|
+
* fenced on the token the purge judged and applied by the store only to that version of the row. Each retirement
|
|
40
|
+
* files a pointer in the due index under the day its tombstone's grace ends, so `scan: 'index'` purges as well as
|
|
41
|
+
* retires; nothing on the row records that day, which the purge derives from the stamp. The purge removes every pointer
|
|
42
|
+
* it read to the row, and a pointer that outlives its row anyway (a purge that ran with another grace, a delete that
|
|
43
|
+
* landed and lost its response, a pointer older than an index scan's lookback) is removed by the next scan that reads
|
|
44
|
+
* it: an index scan in its buckets, and an unscoped fleet scan, which reads every pointer, in all of them. Where a
|
|
45
|
+
* delete only rewrites the row as a tombstone, none of this is done: nothing is removed for good, so a pointer would
|
|
46
|
+
* only add a row for every scan to read.
|
|
36
47
|
*/
|
|
37
48
|
import { type IAuditSink } from './audit.js';
|
|
38
49
|
import type { DropDeps, DropResult } from './erasure.js';
|
|
39
50
|
/** Default cap on retirements per sweep — a bounded batch, so a policy mistake costs one batch, not the fleet. */
|
|
40
51
|
export declare const DEFAULT_RETIRE_LIMIT = 100;
|
|
52
|
+
/**
|
|
53
|
+
* How many purges in a row the registry may refuse in one call before the sweep stops purging for the rest of it. A
|
|
54
|
+
* blanket refusal (a policy that denies delete) costs this many attempts and no more; a refusal particular to one row
|
|
55
|
+
* (a blob with a snapshot, an object under a legal hold) does not hold the purges behind it, since a purge that succeeds
|
|
56
|
+
* starts the count again.
|
|
57
|
+
*/
|
|
58
|
+
export declare const MAX_CONSECUTIVE_PURGE_FAULTS = 3;
|
|
41
59
|
/**
|
|
42
60
|
* Default delay before a retirement's own tombstone row is purged: 24 h.
|
|
43
61
|
*
|
|
@@ -54,7 +72,11 @@ export interface RetireExpiredOptions {
|
|
|
54
72
|
* caller, never from the platform — `store.retireExpired()` fills it in from the store's clock.
|
|
55
73
|
*/
|
|
56
74
|
readonly now: number;
|
|
57
|
-
/**
|
|
75
|
+
/**
|
|
76
|
+
* Maximum segments to retire in this cycle (default 100). It also bounds the purges attempted (a purge the registry
|
|
77
|
+
* refuses is not charged; {@link RetireExpiredResult.purgeFaults} says how purging stops after refusals) and the
|
|
78
|
+
* due-index pointers to nothing one call removes.
|
|
79
|
+
*/
|
|
58
80
|
readonly limit?: number;
|
|
59
81
|
/** Report what would be retired and change nothing. */
|
|
60
82
|
readonly dryRun?: boolean;
|
|
@@ -75,6 +97,11 @@ export interface RetireExpiredOptions {
|
|
|
75
97
|
* pointer write failed (`indexed: false`), or whose ref is too long to index, has no pointer — so a
|
|
76
98
|
* deployment that *only* ever runs `'index'` will never retire those. Run `'fleet'` periodically as the
|
|
77
99
|
* repair pass. The default is `'fleet'` because it is complete by construction.
|
|
100
|
+
*
|
|
101
|
+
* An index scan purges too, on a registry that reports `conditionalDelete`: each retirement files a pointer to its
|
|
102
|
+
* tombstone under the day the tombstone's grace ends, and the scan reads it back with the expiry pointers. A pointer
|
|
103
|
+
* whose segment is gone is removed once read, by an index scan in the buckets it reads, and by an unscoped `'fleet'`
|
|
104
|
+
* scan (no `namespace`) in every bucket.
|
|
78
105
|
*/
|
|
79
106
|
readonly scan?: 'fleet' | 'index';
|
|
80
107
|
/**
|
|
@@ -94,7 +121,8 @@ export interface RetireExpiredOptions {
|
|
|
94
121
|
*/
|
|
95
122
|
readonly lookbackBuckets?: number;
|
|
96
123
|
/**
|
|
97
|
-
* Whether to delete the tombstone rows this sweep's own past retirements left (default `true`)
|
|
124
|
+
* Whether to delete the tombstone rows this sweep's own past retirements left (default `true`): removed for good on
|
|
125
|
+
* a registry that reports `conditionalDelete`, tombstoned on one that does not. Set `false` to
|
|
98
126
|
* keep every one of them — the right choice if something outside this library treats the presence of a
|
|
99
127
|
* `destroyed` row as an attestation. That includes the row of a retirement whose drop found no Storage
|
|
100
128
|
* generation to delete and left none behind: with the default that row is deleted in the same pass, since it
|
|
@@ -179,6 +207,23 @@ export interface RetireExpiredResult {
|
|
|
179
207
|
readonly tombstonesPurged: number;
|
|
180
208
|
/** True when `limit` cut the cycle short — **more segments are still eligible**. Re-run. */
|
|
181
209
|
readonly limited: boolean;
|
|
210
|
+
/**
|
|
211
|
+
* Deletes this sweep attempted that the registry **refused for a reason other than a lost race**: a tombstone's
|
|
212
|
+
* purge, or the removal of a due-index pointer. A policy that denies delete, an Azure blob with a snapshot, or any
|
|
213
|
+
* raw provider error is one; a write that landed between the sweep's read and its fenced delete is not (that is
|
|
214
|
+
* `failed: contended` in the ledger). Each leaves its row or pointer in place, so a purge that keeps failing never
|
|
215
|
+
* frees the name.
|
|
216
|
+
*
|
|
217
|
+
* A refused purge is not charged to `limit`, so retirements go on: a tombstone that cannot be purged does not hold
|
|
218
|
+
* the segments behind it past their expiry. **Purging stops for the rest of the call after
|
|
219
|
+
* {@link MAX_CONSECUTIVE_PURGE_FAULTS} (3) refused purges in a row**, and a purge that succeeds starts the count
|
|
220
|
+
* again. A blanket refusal therefore costs three attempts a call, and a refusal particular to one row (a blob with a
|
|
221
|
+
* snapshot, an object under a legal hold) holds nothing behind it. The next call tries again. Check this field; a
|
|
222
|
+
* ledger entry for each is `skipped`, but a caller that reads only `retired` sees none of it.
|
|
223
|
+
*/
|
|
224
|
+
readonly purgeFaults: number;
|
|
225
|
+
/** The first of those faults, as a ledger reason (`failed: …` with the provider's message). Absent when there were none. */
|
|
226
|
+
readonly firstPurgeFault?: `failed: ${string}`;
|
|
182
227
|
readonly dryRun: boolean;
|
|
183
228
|
/** Per-segment ledger. Inspect it: a `skipped` or failed entry is a segment that still holds data. */
|
|
184
229
|
readonly entries: readonly RetireEntry[];
|
package/dist/core/retry.d.ts
CHANGED
|
@@ -61,6 +61,20 @@ export interface RetryDeps {
|
|
|
61
61
|
export declare function backoffDelayMs(policy: RetryPolicy, attempt: number): number;
|
|
62
62
|
/** Apply the policy's jitter to a computed delay using the injected RNG. */
|
|
63
63
|
export declare function applyJitter(policy: RetryPolicy, delayMs: number, rng: Rng): number;
|
|
64
|
+
/**
|
|
65
|
+
* The store's read retry, as a write path takes it for the reads it makes along the way: a policy (the default when
|
|
66
|
+
* absent) and what {@link withRetry} needs to run it.
|
|
67
|
+
*/
|
|
68
|
+
export type ReadRetry = RetryDeps & {
|
|
69
|
+
readonly policy?: RetryPolicy;
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* Run a read under `retry`, or once when there is none. For the reads a write makes along the way (a load guard's
|
|
73
|
+
* read of the current generation, an erasure's reads of the generations it rewrites and verifies): a read is safe
|
|
74
|
+
* to repeat, so a transient fault on one is retried under the store's policy rather than failing the whole write.
|
|
75
|
+
* The write itself is never passed here.
|
|
76
|
+
*/
|
|
77
|
+
export declare function retryRead<T>(op: () => Promise<T>, retry: ReadRetry | undefined): Promise<T>;
|
|
64
78
|
/**
|
|
65
79
|
* Run `op`, retrying transient failures per `policy`. Resolves with `op`'s result, or rejects with the last
|
|
66
80
|
* error once attempts are exhausted (or immediately for a non-retryable error). The thrown error is always
|
package/dist/core/rollback.d.ts
CHANGED
|
@@ -17,17 +17,32 @@
|
|
|
17
17
|
* Art. 30 record or an incident review wants to find.
|
|
18
18
|
*/
|
|
19
19
|
import { type IAuditSink } from './audit.js';
|
|
20
|
-
import type {
|
|
20
|
+
import type { IKeystore } from './crypto.js';
|
|
21
|
+
import type { IStorageDriver, GenerationMetadata, IRegistryDriver, SegmentRef } from './ports.js';
|
|
21
22
|
/** What the generation helpers need: the objects, and the pointer that says which one is current. */
|
|
22
23
|
export interface GenerationListDeps {
|
|
23
24
|
readonly storage: IStorageDriver;
|
|
24
25
|
readonly registry: IRegistryDriver;
|
|
26
|
+
/**
|
|
27
|
+
* The key of an encrypted segment, so a rollback can read its target's count and metadata and seal them into the
|
|
28
|
+
* row. Without it, a rollback of an encrypted segment still works and leaves the row with no summary of the
|
|
29
|
+
* generation it moves to, which a reader then takes from the object.
|
|
30
|
+
*/
|
|
31
|
+
readonly keystore?: IKeystore;
|
|
25
32
|
}
|
|
26
33
|
/** One generation present in the bucket. */
|
|
27
34
|
export interface GenerationEntry {
|
|
28
35
|
readonly generation: number;
|
|
29
36
|
/** Whether the registry pointer currently names this generation. */
|
|
30
37
|
readonly current: boolean;
|
|
38
|
+
/**
|
|
39
|
+
* The current generation's id count, from its registry row, with no read of the object. Present only on the current
|
|
40
|
+
* entry, and only when the row's summary can be used: for an encrypted segment that takes a keystore that opens the
|
|
41
|
+
* segment's key. It is the row's word, not confirmed against the object.
|
|
42
|
+
*/
|
|
43
|
+
readonly cardinality?: number;
|
|
44
|
+
/** The metadata the current generation was loaded with, from the same row; absent when it has none. */
|
|
45
|
+
readonly metadata?: GenerationMetadata;
|
|
31
46
|
}
|
|
32
47
|
/**
|
|
33
48
|
* Every generation still present for a segment, ascending, with the current one marked.
|
|
@@ -38,7 +53,8 @@ export interface GenerationEntry {
|
|
|
38
53
|
*
|
|
39
54
|
* One registry read and one `list` call, whether or not the segment has a registry row, so it also finds the
|
|
40
55
|
* objects a purged row left behind. It does not open the objects, so it costs nothing per generation and tells you
|
|
41
|
-
* nothing about their contents — read the sizes through the store's own report if you need them.
|
|
56
|
+
* nothing about their contents — read the sizes through the store's own report if you need them. The current entry
|
|
57
|
+
* carries the id count and metadata its row records, from the same read.
|
|
42
58
|
*/
|
|
43
59
|
export declare function listGenerations(ref: SegmentRef, deps: GenerationListDeps): Promise<GenerationEntry[]>;
|
|
44
60
|
export interface RollbackResult {
|
|
@@ -58,12 +74,26 @@ export interface RollbackResult {
|
|
|
58
74
|
* design exists to avoid;
|
|
59
75
|
* - a **crypto-shredded** segment is refused: every generation of it is unreadable, so a rollback would produce a
|
|
60
76
|
* segment that resolves and then fails;
|
|
77
|
+
* - a target that is **not what the row says the segment is** is refused with `IntegrityError`: a cleartext object
|
|
78
|
+
* under a row with keys, or an encrypted one under a row with none, either of which every read would then refuse.
|
|
79
|
+
* One tail read of the target decides it, as does a footer that fails its own checks;
|
|
61
80
|
* - rolling to the generation **already current** is a no-op that reports itself, not an error.
|
|
62
81
|
*
|
|
82
|
+
* The same tail read opens the target when the segment's key is at hand, and the rollback writes the target's own id
|
|
83
|
+
* count and metadata into the row, in the compare-and-swap that moves the pointer: a reader that sees the target as
|
|
84
|
+
* current sees what describes it. (One tail read, and a range read as well when the target's index is longer than the
|
|
85
|
+
* tail read: a target of very many chunks.) A store with no keystore, or one that cannot open the segment's key for any
|
|
86
|
+
* reason, a keystore that is unreachable or timed out included, still rolls an encrypted segment back and leaves the
|
|
87
|
+
* row with no summary of the target. The undo of a rollback whose target was collected meanwhile puts back the summary
|
|
88
|
+
* the row had, and so does the undo of an `allowForward` rollback whose target was replaced meanwhile: an object above
|
|
89
|
+
* the pointer can be deleted by an erasure and its number taken by a load, between the read and the swap, so after the
|
|
90
|
+
* swap such a target is read once more, and the pointer put back unless it is the object whose summary was written.
|
|
91
|
+
*
|
|
63
92
|
* What it does **not** do is delete anything. The generations above the new pointer stay in the bucket, which is
|
|
64
93
|
* what makes the rollback reversible — roll forward again by naming one of them. They are also then *above*
|
|
65
|
-
* `currentGen`, where generation collection never looks. They stay until
|
|
66
|
-
* then keeps the newest `keep` of what is below its
|
|
94
|
+
* `currentGen`, where generation collection never looks. They stay until loads pass them (the first load whose
|
|
95
|
+
* number one of them holds numbers above them all, and collection then keeps the newest `keep` of what is below its
|
|
96
|
+
* pointer), `dropSegment` deletes them, or a subject erasure
|
|
67
97
|
* deletes them: all of those present when it rewrites, and only those holding the id when the current generation does not.
|
|
68
98
|
* Not collecting them is a deliberate trade: an operator who has just undone a bad load should not have the
|
|
69
99
|
* evidence collected out from under them, while a rollback target that still holds erased data would make the
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A generation's summary on its registry row: how a writer builds it, how a reader opens it, and how it is held
|
|
3
|
+
* against the object it describes.
|
|
4
|
+
*
|
|
5
|
+
* The row carries a cached description of the current generation, `{ generation, cardinality, metadata? }`, written by
|
|
6
|
+
* the same compare-and-swap that moves the pointer. On a cleartext segment the values are in the clear. On an
|
|
7
|
+
* encrypted one the row holds `{ generation, sealed }` instead: the cardinality as a fixed-width little-endian u64,
|
|
8
|
+
* then the metadata's canonical JSON if there is any, sealed under the segment's data key with the summary scope of
|
|
9
|
+
* the associated data, which binds it to its namespace, segment and generation. The count is fixed-width so the
|
|
10
|
+
* ciphertext's length reveals only the metadata's size, never how many digits the count has. Pure: no I/O.
|
|
11
|
+
*
|
|
12
|
+
* The generation's `.crbm` object stays the truth, and the summary is a copy of it. {@link summaryAgrees} is the
|
|
13
|
+
* comparison of one with the other.
|
|
14
|
+
*/
|
|
15
|
+
import type { Aead } from './crypto.js';
|
|
16
|
+
import type { ClearRegistrySummary, GenerationMetadata, RegistryRecord, RegistrySummary, SealedRegistrySummary, SegmentRef } from './ports.js';
|
|
17
|
+
/** What a summary says about a generation: its id count, and its metadata when it has any. */
|
|
18
|
+
export interface GenerationDescription {
|
|
19
|
+
readonly cardinality: number;
|
|
20
|
+
readonly metadata?: GenerationMetadata | undefined;
|
|
21
|
+
}
|
|
22
|
+
/** The clear summary of a generation on a cleartext segment. `metadata` is left off when there is none. */
|
|
23
|
+
export declare function clearSummary(generation: number, cardinality: number, metadata?: GenerationMetadata): ClearRegistrySummary;
|
|
24
|
+
/**
|
|
25
|
+
* The sealed summary of a generation on an encrypted segment: the cardinality as a u64, then the metadata's canonical
|
|
26
|
+
* JSON, sealed under `aead` and bound to `ref` and `generation`.
|
|
27
|
+
*/
|
|
28
|
+
export declare function sealSummary(aead: Aead, ref: SegmentRef, generation: number, cardinality: number, metadata?: GenerationMetadata): SealedRegistrySummary;
|
|
29
|
+
/**
|
|
30
|
+
* What a sealed summary says, opened with `aead` under the summary scope of `ref` and the summary's own generation.
|
|
31
|
+
* The bytes are untrusted: a blob that is not base64, is the wrong size, fails its authentication (tampered, or moved
|
|
32
|
+
* from another generation's row), or holds a count outside 0 to 2^32 or metadata that breaks the rules, is an
|
|
33
|
+
* {@link IntegrityError}.
|
|
34
|
+
*/
|
|
35
|
+
export declare function openSummary(aead: Aead, ref: SegmentRef, summary: SealedRegistrySummary): GenerationDescription;
|
|
36
|
+
/**
|
|
37
|
+
* What a row's summary says of its current generation, or `undefined` when it says nothing a reader may use. It is
|
|
38
|
+
* used only for the generation it names, on an active row, and in the shape the row's keys call for: clear on a row
|
|
39
|
+
* with no wrapped keys, sealed on a row with them. A sealed one is opened with `aead`, and is unusable without it,
|
|
40
|
+
* or when it does not open, so a summary a tamperer moved or rewrote is never believed. Never throws for a bad
|
|
41
|
+
* summary: the row's copy is a cache, and a copy that cannot be used is a copy that is not there.
|
|
42
|
+
*/
|
|
43
|
+
export declare function usableSummary(ref: SegmentRef, row: RegistryRecord, aead: Aead | undefined): GenerationDescription | undefined;
|
|
44
|
+
/**
|
|
45
|
+
* Whether a summary's description of a generation is what its object holds: the same id count and the same metadata,
|
|
46
|
+
* compared in canonical form, so key order never matters. A summary with metadata over an object with none, and the
|
|
47
|
+
* reverse, disagree: whether an encrypted object's metadata block is present is not authenticated (stripping it needs
|
|
48
|
+
* no key), so the row's sealed copy is what a missing block is held against.
|
|
49
|
+
*/
|
|
50
|
+
export declare function summaryAgrees(summary: GenerationDescription, object: GenerationDescription): boolean;
|
|
51
|
+
/**
|
|
52
|
+
* The metadata an erasure carries into its rewrite of a generation: the object's own, except that an object with none
|
|
53
|
+
* whose row's authenticated summary of the same generation has some gets the row's. Whether an encrypted object has a
|
|
54
|
+
* metadata block is not authenticated, so a block stripped from one reads as none, and the sealed summary on the row is
|
|
55
|
+
* the one copy that says there was a block; carrying it keeps the rewrite from erasing the record of what it dropped.
|
|
56
|
+
* Pass `authenticated` only for a summary that was opened (a clear one has no more authority than the block it sits
|
|
57
|
+
* beside). Never refuses: a rewrite over a stripped block is still an erasure.
|
|
58
|
+
*/
|
|
59
|
+
export declare function metadataToCarry(object: GenerationMetadata | undefined, authenticated: GenerationDescription | undefined): GenerationMetadata | undefined;
|
|
60
|
+
/** The summary of a generation of `ref`, clear or sealed under `aead` as the segment's keys call for. */
|
|
61
|
+
export declare function summaryOf(ref: SegmentRef, generation: number, description: GenerationDescription, aead: Aead | undefined): RegistrySummary;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The registry token's incarnation: the part of a token that says which incarnation of a name a row is.
|
|
3
|
+
*
|
|
4
|
+
* A shipped registry gives a row it creates a token `<incarnation>.<counter>.<write>`: 32 lowercase hex digits drawn
|
|
5
|
+
* from injected entropy when the row is created, a decimal counter, and 16 hex digits drawn for each write. Every write
|
|
6
|
+
* of the row keeps the incarnation, and a name created again draws a new one, so two incarnations are told apart by it
|
|
7
|
+
* even when their rows were created in the same millisecond or one is gone. A row a 0.11 build wrote has no incarnation
|
|
8
|
+
* for as long as it lives (`"7"`, then `"8.<write>"`), and a registry of someone else's issues tokens in any form.
|
|
9
|
+
*
|
|
10
|
+
* This lives in `core/` so the publish can read it, and the registry drivers, which build the tokens, import it from
|
|
11
|
+
* here: the form has one definition.
|
|
12
|
+
*/
|
|
13
|
+
import type { Token } from './ports.js';
|
|
14
|
+
/** An incarnation-form token: the incarnation id, the counter and a write part, `.`-separated, in lowercase hex. */
|
|
15
|
+
export declare const INCARNATION_TOKEN: RegExp;
|
|
16
|
+
/**
|
|
17
|
+
* The incarnation id in a token, or `undefined` for one that has none: a schema-1-born row's, or any token another
|
|
18
|
+
* driver issues. Two tokens with the same incarnation are writes of one incarnation of a row.
|
|
19
|
+
*/
|
|
20
|
+
export declare function incarnationOf(token: Token): string | undefined;
|
|
21
|
+
/**
|
|
22
|
+
* Whether two reads of one name's row are one incarnation of it.
|
|
23
|
+
*
|
|
24
|
+
* - Both tokens carry an incarnation id: the ids decide, and the clock stamps do not enter.
|
|
25
|
+
* - Only one does: different incarnations. A name created again is always given an incarnation id, and a write never
|
|
26
|
+
* takes one away, so a row without an id never becomes a row with one, nor the reverse.
|
|
27
|
+
* - Neither does (a row 0.11 wrote, or a registry of someone else's): there is no id to compare, and the creation stamp
|
|
28
|
+
* is the only signal left. It is a clock reading, so a caller that acts on this answer proves its own object too.
|
|
29
|
+
*/
|
|
30
|
+
export declare function sameIncarnation(a: {
|
|
31
|
+
readonly token: Token;
|
|
32
|
+
readonly createdAt: number;
|
|
33
|
+
}, b: {
|
|
34
|
+
readonly token: Token;
|
|
35
|
+
readonly createdAt: number;
|
|
36
|
+
}): boolean;
|
package/dist/core/validate.d.ts
CHANGED
|
@@ -7,6 +7,8 @@ import type { ChunkRef, SegmentRef } from './ports.js';
|
|
|
7
7
|
* a namespace and a segment at the ceiling.
|
|
8
8
|
*/
|
|
9
9
|
export declare const MAX_NAME_LENGTH = 256;
|
|
10
|
+
/** Whether `value` is well-formed UTF-16: no unpaired surrogate, so it survives a round trip through UTF-8. */
|
|
11
|
+
export declare function isWellFormedString(value: string): boolean;
|
|
10
12
|
/**
|
|
11
13
|
* How long a name is once storage has it — the longer of the two encodings.
|
|
12
14
|
*
|