@cloudbitmaps/core 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -8
- package/dist/chunk-7GT632UF.js +641 -0
- package/dist/chunk-7GT632UF.js.map +7 -0
- package/dist/chunk-CGILDXGP.js +73 -0
- package/dist/chunk-CGILDXGP.js.map +7 -0
- package/dist/chunk-GKHERVVQ.js +121 -0
- package/dist/chunk-GKHERVVQ.js.map +7 -0
- package/dist/{chunk-IGLIZBR7.js → chunk-MVUD4R72.js} +253 -765
- package/dist/chunk-MVUD4R72.js.map +7 -0
- package/dist/core/audit.d.ts +53 -15
- package/dist/core/bit-route.d.ts +9 -0
- package/dist/core/blob.d.ts +4 -0
- package/dist/core/budget.d.ts +8 -3
- package/dist/core/check.d.ts +27 -0
- package/dist/core/codec.d.ts +1 -1
- package/dist/core/combine-expr.d.ts +9 -5
- package/dist/core/combine-feed.d.ts +11 -3
- package/dist/core/combine-many.d.ts +28 -13
- package/dist/core/consistency.d.ts +11 -2
- package/dist/core/cooperative.d.ts +7 -0
- package/dist/core/crbm-storage-source.d.ts +7 -4
- package/dist/core/crypto.d.ts +5 -0
- package/dist/core/determinism.d.ts +2 -1
- package/dist/core/discover.d.ts +5 -3
- package/dist/core/engine.d.ts +4 -4
- package/dist/core/erase-id.d.ts +52 -19
- package/dist/core/erasure.d.ts +62 -28
- package/dist/core/errors.d.ts +83 -9
- package/dist/core/leases.d.ts +2 -0
- package/dist/core/load-input.d.ts +48 -0
- package/dist/core/load.d.ts +71 -16
- package/dist/core/metrics.d.ts +22 -1
- package/dist/core/pinned-storage-source.d.ts +27 -8
- package/dist/core/pointer-id.d.ts +9 -0
- package/dist/core/ports.d.ts +107 -13
- package/dist/core/preflight.d.ts +80 -0
- package/dist/core/registry-scan.d.ts +0 -5
- package/dist/core/retention-sweep.d.ts +61 -16
- package/dist/core/retention.d.ts +18 -7
- package/dist/core/retry.d.ts +5 -2
- package/dist/core/rollback.d.ts +23 -12
- package/dist/core/validate.d.ts +33 -2
- package/dist/driver-kit.d.ts +9 -4
- package/dist/driver-kit.js +63 -27
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +24 -1
- package/dist/drivers/_shared/options.d.ts +15 -7
- package/dist/drivers/backends.d.ts +19 -2
- package/dist/drivers/crypto.d.ts +18 -3
- package/dist/export/index.d.ts +25 -10
- package/dist/index.d.ts +19 -17
- package/dist/index.js +2457 -2283
- package/dist/index.js.map +4 -4
- package/dist/node.d.ts +14 -0
- package/dist/node.js +1064 -0
- package/dist/node.js.map +7 -0
- package/package.json +16 -2
- package/dist/chunk-IGLIZBR7.js.map +0 -7
- package/dist/core/absent-operands.d.ts +0 -8
- package/dist/core/bulk-load.d.ts +0 -148
- package/dist/core/cas-retry.d.ts +0 -27
- package/dist/core/chunk-checks.d.ts +0 -39
- package/dist/core/chunk-stream.d.ts +0 -22
- package/dist/core/chunk-window.d.ts +0 -37
- package/dist/core/combine-ledger.d.ts +0 -43
- package/dist/core/crbm/crc32c.d.ts +0 -10
- package/dist/core/crbm/fingerprint.d.ts +0 -23
- package/dist/core/crbm/format.d.ts +0 -150
- package/dist/core/crbm/frame.d.ts +0 -10
- package/dist/core/crbm/plan-reads.d.ts +0 -50
- package/dist/core/crbm/varint.d.ts +0 -14
- package/dist/core/crbm/writer.d.ts +0 -72
- package/dist/core/crbm-open.d.ts +0 -154
- package/dist/core/crbm-write.d.ts +0 -75
- package/dist/core/due-index.d.ts +0 -53
- package/dist/core/generation-gc.d.ts +0 -293
- package/dist/core/generation-list.d.ts +0 -11
- package/dist/core/id-window.d.ts +0 -46
- package/dist/core/invalidation-scope.d.ts +0 -77
- package/dist/core/item-pull.d.ts +0 -29
- package/dist/core/known-generations-shape.d.ts +0 -13
- package/dist/core/known-generations.d.ts +0 -147
- package/dist/core/ledger-seed.d.ts +0 -17
- package/dist/core/metadata.d.ts +0 -42
- package/dist/core/order.d.ts +0 -2
- package/dist/core/publish.d.ts +0 -207
- package/dist/core/reader-defaults.d.ts +0 -29
- package/dist/core/summary.d.ts +0 -110
- package/dist/core/token.d.ts +0 -41
- package/dist/drivers/_shared/entropy.d.ts +0 -5
- package/dist/drivers/_shared/registry.d.ts +0 -107
- package/dist/drivers/localfs/exact-case.d.ts +0 -34
- package/dist/drivers/localfs/fs-util.d.ts +0 -55
- package/dist/drivers/localfs/paths.d.ts +0 -40
- package/dist/drivers/localfs/registry.d.ts +0 -50
- package/dist/drivers/localfs/storage.d.ts +0 -28
- package/dist/drivers/memory.d.ts +0 -64
- package/dist/testing/fuzz-core.d.ts +0 -15
package/dist/core/erase-id.d.ts
CHANGED
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
* There is no per-id delete on an immutable object, and there is no mutable tier to hold a tombstone, so erasure
|
|
6
6
|
* is what every other write in this library is: a new generation. The current generation is streamed chunk by
|
|
7
7
|
* chunk through the ascending writer — every chunk decoded, range-checked and re-encoded, the one holding the id
|
|
8
|
-
* with that bit cleared — then published fenced on the generation it streamed, and that generation is collected
|
|
9
|
-
* (`keep: 0`), so the bit is **physically gone from the bucket when this
|
|
10
|
-
*
|
|
8
|
+
* with that bit cleared — then published fenced on the generation it streamed, and that generation is collected
|
|
9
|
+
* immediately (`keep: 0`), so when it answers `erased: true` the bit is **physically gone from the bucket when this
|
|
10
|
+
* returns** (a noncurrent version, a replica or a backup keeps the old object until its own lifecycle removes it).
|
|
11
|
+
* Bounded memory: a window of 32 chunk reads ahead of the writer, never the whole segment.
|
|
11
12
|
*
|
|
12
13
|
* The rewrite is the same generation without one id, so it keeps everything else: the new object carries the source's
|
|
13
14
|
* metadata as it is, and the row's summary of it, built from what was written, counts one id fewer and holds the same
|
|
@@ -21,12 +22,17 @@
|
|
|
21
22
|
* leaves generations above the pointer that were once current and can be made current again, so a holder can sit
|
|
22
23
|
* above the pointer as well as below it, and there can be several. Before it reports `erased: true` the call
|
|
23
24
|
* therefore lists the bucket once more and reads every generation it has not already seen to be clean; any that
|
|
24
|
-
* still holds the id makes it report or throw instead.
|
|
25
|
-
*
|
|
25
|
+
* still holds the id makes it report or throw instead. A number can be taken again once its object is deleted, so a
|
|
26
|
+
* generation it saw clean is read again too, but for two: the one it found current and its own rewrite, each skipped
|
|
27
|
+
* only while the row, read after that listing, names it with the footer checksum of the object the call read or wrote.
|
|
28
|
+
* In the ordinary case that list returns one generation, the current one, which the row names: one `list` and one
|
|
29
|
+
* read of the row.
|
|
26
30
|
*
|
|
27
31
|
* What it does NOT reach: backups, replicas and noncurrent object versions hold the old object until their own
|
|
28
|
-
* lifecycle removes it. For an at-rest guarantee that
|
|
29
|
-
* (`destroySegment`), which is per segment — a per-id shred is infeasible because one DEK covers the whole segment
|
|
32
|
+
* lifecycle removes it. For an at-rest guarantee that reaches those, use encryption plus crypto-shred
|
|
33
|
+
* (`destroySegment`), which is per segment — a per-id shred is infeasible because one DEK covers the whole segment —
|
|
34
|
+
* and is complete once no retained copy of the segment's registry row holds its wrapped key, or every KEK that
|
|
35
|
+
* wrapped it is destroyed.
|
|
30
36
|
*
|
|
31
37
|
* Concurrency. This is a **read-modify-write**, not a plain forward-only publish, and that distinction is where
|
|
32
38
|
* its correctness lives: the new generation is `from` minus one bit, so it is a valid successor to `from` and to
|
|
@@ -36,8 +42,8 @@
|
|
|
36
42
|
* incarnation of a name, and a name that is retired and re-created restarts at `0`, so the number alone matched
|
|
37
43
|
* a different segment entirely. Anything else — a load that published in the meantime, or another
|
|
38
44
|
* erasure that got there first — is reported as `reason: 'superseded'`, `erased: false`, and the caller re-runs
|
|
39
|
-
* against the new generation.
|
|
40
|
-
* `WriteConflictError
|
|
45
|
+
* against the new generation. So is a writer that claims the same generation *number* first (write-once): the put
|
|
46
|
+
* is refused with a `WriteConflictError`, which is read here as the lost race it is.
|
|
41
47
|
*
|
|
42
48
|
* The same answer covers the read half, and for the same reason. An erasure collects with `keep: 0`, which takes
|
|
43
49
|
* every generation below its new pointer — so a racing erasure can delete the generation this call is streaming,
|
|
@@ -48,8 +54,8 @@
|
|
|
48
54
|
* where the truth was to re-run.
|
|
49
55
|
*
|
|
50
56
|
* Which reason comes from **re-reading the row**, not from assuming a supersession: a moved pointer is
|
|
51
|
-
* `'superseded'`, a tombstoned row `'destroyed'`, a purged row `'absent'`, a row
|
|
52
|
-
* `'no-
|
|
57
|
+
* `'superseded'`, a tombstoned row `'destroyed'`, a purged row `'absent'`, a row that lost its pointer
|
|
58
|
+
* `'superseded'` too (its bucket was not searched, so no claim about it is made; a re-run searches it). A pointer still on `from` means the object it names is genuinely gone — the forbidden
|
|
53
59
|
* `missing-storage-generation` state — or is another object than the row's summary names by its fingerprint, and that
|
|
54
60
|
* throws, because no re-run fixes either. That `NotFoundError` is the only signal of those states, so a faulting
|
|
55
61
|
* re-read rethrows it rather than replacing it with a transient-looking registry error.
|
|
@@ -102,7 +108,7 @@
|
|
|
102
108
|
import { type IAuditSink } from './audit.js';
|
|
103
109
|
import type { CodecInterface } from './codec.js';
|
|
104
110
|
import type { Yielder } from './cooperative.js';
|
|
105
|
-
import type { Rng } from './determinism.js';
|
|
111
|
+
import type { Entropy, Rng } from './determinism.js';
|
|
106
112
|
import type { IKeystore } from './crypto.js';
|
|
107
113
|
import type { IStorageDriver, IRegistryDriver, SegmentRef } from './ports.js';
|
|
108
114
|
import { type ReadRetry } from './retry.js';
|
|
@@ -148,6 +154,13 @@ export interface EraseIdDeps {
|
|
|
148
154
|
* renewal. Absent, the read retry's source is used if it has one, and otherwise each wait is its bound.
|
|
149
155
|
*/
|
|
150
156
|
readonly rng?: Rng;
|
|
157
|
+
/**
|
|
158
|
+
* The source of the rewrite's identity: 12 random bytes its footer carries when the segment is cleartext (an encrypted
|
|
159
|
+
* rewrite carries its index's AEAD nonce), so the rewrite is not the same object as a load of the same ids at its
|
|
160
|
+
* number. Required: an erasure without one (a JavaScript caller can leave it out) is refused with
|
|
161
|
+
* {@link ValidationError} before any read, keystore call, delete or write, even when it would find nothing to rewrite.
|
|
162
|
+
*/
|
|
163
|
+
readonly entropy: Entropy;
|
|
151
164
|
/** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. The `.crbm` reader refuses an entry above its own `maxPayloadBytes` (1 MiB, plus 28 bytes when encrypted) at open, so raise that on the chunk source too. */
|
|
152
165
|
readonly maxBitmapBytes?: number;
|
|
153
166
|
/**
|
|
@@ -169,9 +182,10 @@ export interface EraseIdResult {
|
|
|
169
182
|
*/
|
|
170
183
|
readonly erased: boolean;
|
|
171
184
|
/**
|
|
172
|
-
* Why the id was not erased, when `erased` is false. `'absent'` (no registry row), `'destroyed'` (a
|
|
173
|
-
*
|
|
174
|
-
* objects, none of them found to hold the id, and `collected` names any this
|
|
185
|
+
* Why the id was not erased, when `erased` is false. `'absent'` (no registry row), `'destroyed'` (a tombstone; its
|
|
186
|
+
* objects are searched first), `'no-generation'` (a row with no generation yet when the call read it, its first
|
|
187
|
+
* read: its bucket can hold first loads' objects, none of them found to hold the id, and `collected` names any this
|
|
188
|
+
* call deleted unsearched, see below),
|
|
175
189
|
* `'not-member'` (no
|
|
176
190
|
* generation in the bucket holds the id — the common case across a fleet scan — with the same note on `collected`),
|
|
177
191
|
* or `'superseded'`.
|
|
@@ -188,10 +202,12 @@ export interface EraseIdResult {
|
|
|
188
202
|
* another erasure of the *same* id, the re-run reports `'not-member'` because it is already gone. Either way the
|
|
189
203
|
* re-run settles it, which is why it is the documented action for this reason and for no other.
|
|
190
204
|
*
|
|
191
|
-
*
|
|
205
|
+
* `'absent'` and `'destroyed'` are also returned when the row changes *underneath* a call already in flight, not
|
|
192
206
|
* only when it is read up front — a concurrent `dropSegment` gives `'destroyed'`, once the tombstone's objects have
|
|
193
|
-
* been searched as a fresh call searches them, and a retention sweep that purges the row gives `'absent'`.
|
|
194
|
-
*
|
|
207
|
+
* been searched as a fresh call searches them, and a retention sweep that purges the row gives `'absent'`.
|
|
208
|
+
* `'no-generation'` is only the answer to a row read with no generation: a row that loses its pointer *after* the
|
|
209
|
+
* call found one (a purge and re-creation, or a hand-made write) is `'superseded'`, because the call found a holder
|
|
210
|
+
* in that bucket and stopped before deleting it, and a re-run searches it again.
|
|
195
211
|
*/
|
|
196
212
|
readonly reason?: 'absent' | 'destroyed' | 'no-generation' | 'not-member' | 'superseded';
|
|
197
213
|
/**
|
|
@@ -222,7 +238,7 @@ export interface EraseIdResult {
|
|
|
222
238
|
* a missing holder as "someone else got there", never as a failure.
|
|
223
239
|
*
|
|
224
240
|
* A call that cannot establish the claim **throws** `WriteConflictError` or the underlying fault rather than
|
|
225
|
-
* report `erased: true` over bytes that are still there. Four ways that happens: a Storage `delete` fault; a
|
|
241
|
+
* report `erased: true` over bytes that are still there. Four ways that happens: a Storage `delete` fault the call cannot settle by reading the object back; a
|
|
226
242
|
* collection pass that could not prove the segment was still the same one (which an ordinary retirement landing
|
|
227
243
|
* mid-call is enough to cause); the same refusal on the path where the current generation did not hold the id
|
|
228
244
|
* and nothing was published at all; and a generation still holding the id when the bucket is listed at the end
|
|
@@ -232,6 +248,13 @@ export interface EraseIdResult {
|
|
|
232
248
|
* rewrite discards its own object, when it gets no answer that reading the row can settle, throws the registry's
|
|
233
249
|
* `TransientError`, and that object is not deleted.
|
|
234
250
|
*
|
|
251
|
+
* A delete that gets no answer is settled by reading its key: gone, it counts as deleted and is listed in
|
|
252
|
+
* `collected`. Another object under the number, for a delete conditioned on the version searched, is not counted as
|
|
253
|
+
* deleted: the receipt then rests on the listing of the bucket, and says `erased: true` only when that object does
|
|
254
|
+
* not hold the id. When that read fails too, or the delete lands after it, the call reports the fault, and a re-run
|
|
255
|
+
* finds the object gone and lists nothing for the segment: keep the error with the ledger as the record of the
|
|
256
|
+
* deletion.
|
|
257
|
+
*
|
|
235
258
|
* A rollback that lands between the call's read of the row and one of its deletes, onto the generation being deleted,
|
|
236
259
|
* leaves the pointer on an object that is gone. The call finds it at its next read of the row, or by reading the row
|
|
237
260
|
* once more after its last delete, and **throws `NotFoundError`** rather than report anything: every read of the
|
|
@@ -267,3 +290,13 @@ export interface EraseIdResult {
|
|
|
267
290
|
export declare function eraseIdFromSegment(ref: SegmentRef, id: number, deps: EraseIdDeps, options?: {
|
|
268
291
|
audit?: IAuditSink;
|
|
269
292
|
}): Promise<EraseIdResult>;
|
|
293
|
+
/** What the search for an id in an object needs of the store: the objects, the codec, and the read bounds. */
|
|
294
|
+
export type SearchDeps = Pick<EraseIdDeps, 'storage' | 'codec' | 'maxBitmapBytes' | 'readRetry'>;
|
|
295
|
+
/**
|
|
296
|
+
* The newest generation left under a tombstone that holds `id` in the clear, or `null`: the search
|
|
297
|
+
* {@link eraseIdFromSegment} makes under a tombstone, without its deletes. The caller has read the row as `destroyed`;
|
|
298
|
+
* this reads no row. It lists the bucket once and opens objects newest first, a few at a time, stopping at the first
|
|
299
|
+
* that holds the id; an object sealed under a key is no holder. It writes and deletes nothing. A fault on a read is
|
|
300
|
+
* thrown, after the read retry in `deps.readRetry` (the listing is retried too), never read as "not found".
|
|
301
|
+
*/
|
|
302
|
+
export declare function findIdUnderTombstone(ref: SegmentRef, id: number, deps: SearchDeps): Promise<number | null>;
|
package/dist/core/erasure.d.ts
CHANGED
|
@@ -1,33 +1,51 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Whole-segment erasure: crypto-shred (`destroySegment`, `eraseNamespace`) and disposal (`dropSegment`).
|
|
3
3
|
*
|
|
4
|
-
* GDPR "right to erasure" on immutable, backed-up storage: you can't delete a `.crbm` object from every
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* GDPR "right to erasure" on immutable, backed-up storage: you can't delete a `.crbm` object from every backup, but you
|
|
5
|
+
* can **delete its key**. `destroySegment` drops the segment's wrapped DEK(s) from its current registry row (a CAS to a
|
|
6
|
+
* `destroyed` tombstone). Every copy of the encrypted Storage bytes, backups and WORM copies included, is then
|
|
7
|
+
* unreadable once no retained copy of the row still holds the wrapped key (a noncurrent version, a backup or a
|
|
8
|
+
* point-in-time copy of the row), or once every KEK that wrapped it is destroyed; until then a registry restore to a
|
|
9
|
+
* point before the shred makes the segment readable again. **Immediate**, and it cannot be undone through the library
|
|
10
|
+
* (a registry restore to before it can), so it's gated behind an explicit confirmation (the
|
|
11
|
+
* segment's name; the namespace is not checked, so build the ref's `namespace` from a value you have checked). A `destroyed` segment reads as empty: no generation resolves through a destroyed row. Only an
|
|
12
|
+
* **encrypted** segment is crypto-shredded; a cleartext one has no key to shred, and `allowCleartext` tombstones it
|
|
13
|
+
* with its objects left readable.
|
|
10
14
|
*
|
|
11
|
-
* The tombstone is also the fence every writer respects: a load refuses
|
|
12
|
-
*
|
|
13
|
-
*
|
|
15
|
+
* The tombstone is also the fence every writer respects: a load refuses a `destroyed` row, so a load racing an erasure
|
|
16
|
+
* cannot resurrect the segment. A single id's erasure is a different operation — `eraseIdFromSegment` rewrites the
|
|
17
|
+
* generation without it.
|
|
14
18
|
*/
|
|
15
19
|
import { type IAuditSink } from './audit.js';
|
|
16
20
|
import type { ChurnDeps } from './leases.js';
|
|
17
21
|
import type { IStorageDriver, IRegistryDriver, RegistryRecord, SegmentRef } from './ports.js';
|
|
22
|
+
/**
|
|
23
|
+
* What the erasure calls need: the registry whose rows they tombstone, and, optionally, the clock and rng they wait out
|
|
24
|
+
* lease churn with.
|
|
25
|
+
*/
|
|
18
26
|
export interface EraseDeps extends ChurnDeps {
|
|
27
|
+
/** The registry holding the segments' rows. */
|
|
19
28
|
readonly registry: IRegistryDriver;
|
|
20
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* What destroying one segment did: a crypto-shred, a cleartext tombstone (the bytes stay readable), nothing for a
|
|
32
|
+
* segment already destroyed or absent, or, with `destroyed: false` and a `reason`, a segment left as it was, which may
|
|
33
|
+
* still hold data. The result of `destroySegment`, and one entry of what `eraseNamespace` returns.
|
|
34
|
+
*/
|
|
21
35
|
export interface DestroyResult {
|
|
36
|
+
/** The segment's name. */
|
|
22
37
|
readonly segment: string;
|
|
38
|
+
/** The segment's namespace, or `undefined` for the default one. */
|
|
23
39
|
readonly namespace?: string;
|
|
24
40
|
/** True iff the segment is now a `destroyed` tombstone (incl. the idempotent already-destroyed case). */
|
|
25
41
|
readonly destroyed: boolean;
|
|
26
42
|
/**
|
|
27
43
|
* True iff **this call** performed a genuine crypto-shred — an encrypted segment whose wrapped DEK(s) were
|
|
28
|
-
* dropped, so its Storage bytes
|
|
29
|
-
*
|
|
30
|
-
*
|
|
44
|
+
* dropped from its current registry row, so its Storage bytes, in any copy, are unreadable once no retained copy
|
|
45
|
+
* of the row holds the key, or every KEK that wrapped it is destroyed. False for a cleartext tombstone (bytes
|
|
46
|
+
* remain readable), for the idempotent already-destroyed case, and for an absent segment. This is what the
|
|
47
|
+
* `segment.erase` audit event keys off — a cleartext "erase" destroys no key. A shred that throws `TransientError`
|
|
48
|
+
* after its read-back also failed may have landed; its re-run answers `'already'` with `cryptoShredded: false`.
|
|
31
49
|
*/
|
|
32
50
|
readonly cryptoShredded: boolean;
|
|
33
51
|
/**
|
|
@@ -38,14 +56,20 @@ export interface DestroyResult {
|
|
|
38
56
|
*
|
|
39
57
|
* The last two come with `destroyed: false` and mean **the segment still holds data**. A namespace erase
|
|
40
58
|
* returns them rather than throwing, so its entries have to be inspected — see {@link eraseNamespace}.
|
|
59
|
+
*
|
|
60
|
+
* A shred that throws `TransientError` after its read-back also failed may have landed; its re-run answers
|
|
61
|
+
* `'already'` with `cryptoShredded: false`.
|
|
41
62
|
*/
|
|
42
63
|
readonly reason?: string;
|
|
43
64
|
}
|
|
44
65
|
/**
|
|
45
|
-
* Crypto-shred one segment.
|
|
46
|
-
*
|
|
47
|
-
* via `allowCleartext` (there's no key to shred; its
|
|
48
|
-
* segment resolving).
|
|
66
|
+
* Crypto-shred one segment. It cannot be undone through the library (a registry restore to a point before it can).
|
|
67
|
+
* `confirmSegment` must equal `ref.segment` (a guard against an accidental destroy of another name; it checks the
|
|
68
|
+
* name only, not the namespace, so build the ref's `namespace` from a value you have checked). A non-encrypted segment is rejected unless you opt in via `allowCleartext` (there's no key to shred; its
|
|
69
|
+
* Storage bytes stay readable — the tombstone only stops the segment resolving).
|
|
70
|
+
*
|
|
71
|
+
* The options are `confirmSegment`, `allowCleartext` and `audit`. Any other key (`dryRun`, which this call does not
|
|
72
|
+
* take, included) and an `allowCleartext` that is not a boolean throw `ValidationError` before the registry is read.
|
|
49
73
|
*/
|
|
50
74
|
export declare function destroySegment(ref: SegmentRef, deps: EraseDeps, options: {
|
|
51
75
|
confirmSegment: string;
|
|
@@ -53,14 +77,16 @@ export declare function destroySegment(ref: SegmentRef, deps: EraseDeps, options
|
|
|
53
77
|
audit?: IAuditSink;
|
|
54
78
|
}): Promise<DestroyResult>;
|
|
55
79
|
/**
|
|
56
|
-
* Crypto-shred every segment in a namespace.
|
|
57
|
-
* Returns a per-segment result (skips cleartext segments
|
|
80
|
+
* Crypto-shred every segment in a namespace. It cannot be undone through the library (a registry restore to a point
|
|
81
|
+
* before it can). `confirmNamespace` must equal `namespace`. Returns a per-segment result (skips cleartext segments
|
|
82
|
+
* unless `allowCleartext`).
|
|
58
83
|
*
|
|
59
84
|
* **Bounded.** It lists the whole namespace before it destroys anything, and holds the listing resident, so the
|
|
60
|
-
* listing is capped at `maxScanSegments` (default
|
|
85
|
+
* listing is capped at `maxScanSegments` (default 250,000), the ceiling every
|
|
61
86
|
* other fleet scan keeps. A namespace over it throws `BudgetExceededError` **before any segment is destroyed**,
|
|
62
87
|
* never part-way through; raise `maxScanSegments` when the namespace really is that large and the memory is there.
|
|
63
88
|
* A `maxScanSegments` that is not a safe whole number of at least 1 throws `ValidationError`, again before anything is destroyed.
|
|
89
|
+
* So does any option it does not take (`dryRun` included) and an `allowCleartext` that is not a boolean.
|
|
64
90
|
*/
|
|
65
91
|
export declare function eraseNamespace(namespace: string, deps: EraseDeps, options: {
|
|
66
92
|
confirmNamespace: string;
|
|
@@ -76,8 +102,11 @@ export declare function eraseNamespace(namespace: string, deps: EraseDeps, optio
|
|
|
76
102
|
export interface DropDeps extends EraseDeps {
|
|
77
103
|
readonly storage: IStorageDriver;
|
|
78
104
|
}
|
|
105
|
+
/** What `dropSegment` did to one segment, or, on a dry run, would do. */
|
|
79
106
|
export interface DropResult {
|
|
107
|
+
/** The segment's name. */
|
|
80
108
|
readonly segment: string;
|
|
109
|
+
/** The segment's namespace, or `undefined` for the default one. */
|
|
81
110
|
readonly namespace?: string;
|
|
82
111
|
/**
|
|
83
112
|
* **True iff this segment is now empty as a result of this call — or already was.** This is the field a
|
|
@@ -111,14 +140,16 @@ export interface DropResult {
|
|
|
111
140
|
readonly wouldDelete?: readonly number[];
|
|
112
141
|
/**
|
|
113
142
|
* On a dry run, whether the real drop would **also** crypto-shred (i.e. the segment is encrypted).
|
|
114
|
-
* `undefined` on a real run. This is the one fact in the preview that
|
|
115
|
-
*
|
|
143
|
+
* `undefined` on a real run. This is the one fact in the preview that reaches every copy of the objects, backups
|
|
144
|
+
* and WORM copies included (once no retained copy of the row holds the key, or every KEK that wrapped it is
|
|
145
|
+
* destroyed) — a dry run that did not surface it would preview the recoverable half only.
|
|
116
146
|
*/
|
|
117
147
|
readonly wouldCryptoShred?: boolean;
|
|
118
148
|
/**
|
|
119
|
-
* True iff the segment was encrypted and its wrapped DEK(s) were dropped
|
|
120
|
-
* are unreadable
|
|
121
|
-
*
|
|
149
|
+
* True iff the segment was encrypted and its wrapped DEK(s) were dropped from its current registry row as part of
|
|
150
|
+
* this call — so its bytes are unreadable in every copy, backups included, once no retained copy of the row holds
|
|
151
|
+
* the key or every KEK that wrapped it is destroyed, not merely deleted from the bucket. Deleting an object does
|
|
152
|
+
* not reach a noncurrent version, a replica, or a PITR snapshot; removing the key does, on those terms.
|
|
122
153
|
*/
|
|
123
154
|
readonly cryptoShredded: boolean;
|
|
124
155
|
/**
|
|
@@ -133,14 +164,17 @@ export interface DropResult {
|
|
|
133
164
|
readonly reason?: string;
|
|
134
165
|
}
|
|
135
166
|
/**
|
|
136
|
-
* **Dispose of a segment: tombstone it, then delete its Storage objects.**
|
|
167
|
+
* **Dispose of a segment: tombstone it, then delete its Storage objects.** It cannot be undone through the library
|
|
168
|
+
* (a restore from the registry's and the bucket's versions or backups can).
|
|
137
169
|
*
|
|
138
170
|
* WHY THIS EXISTS, given {@link destroySegment} already erases. Because `destroySegment` answers a *compliance*
|
|
139
171
|
* question and this one answers an *operational* question, and they are not the same:
|
|
140
172
|
*
|
|
141
|
-
* - `destroySegment` **crypto-shreds** — it
|
|
142
|
-
*
|
|
143
|
-
*
|
|
173
|
+
* - `destroySegment` **crypto-shreds** — it removes the key from the segment's current registry row, so the bytes
|
|
174
|
+
* are unreadable everywhere, immutable backups included, once no retained copy of the row holds the key or every
|
|
175
|
+
* KEK that wrapped it is destroyed: the only erasure that reaches WORM copies of the objects. But it **leaves the
|
|
176
|
+
* objects in the bucket**, still billed, and it *requires* encryption because a cleartext segment has no key to
|
|
177
|
+
* discard.
|
|
144
178
|
* - `dropSegment` **removes the storage**. It works on a cleartext segment, and on an encrypted one it *also*
|
|
145
179
|
* drops the DEKs, so it is a strict superset there.
|
|
146
180
|
*
|
package/dist/core/errors.d.ts
CHANGED
|
@@ -15,15 +15,16 @@
|
|
|
15
15
|
* does not match — and silently, because a `catch` that stops matching just falls through, defeating
|
|
16
16
|
* transient retry and the publish path's conflict handling with no error of its own.
|
|
17
17
|
*
|
|
18
|
-
* `Symbol.for` is identity-stable across copies, bundles and realms, so classify errors with
|
|
19
|
-
*
|
|
18
|
+
* `Symbol.for` is identity-stable across copies, bundles and realms, so classify errors with `errorKind`
|
|
19
|
+
* (never `instanceof`) anywhere an error may cross that boundary.
|
|
20
20
|
*/
|
|
21
21
|
declare const ERROR_BRAND: unique symbol;
|
|
22
22
|
declare const TRANSIENT_BRAND: unique symbol;
|
|
23
23
|
/** Base class for every error CloudRoaring throws. */
|
|
24
24
|
export declare class CloudRoaringError extends Error {
|
|
25
|
-
/** Cross-bundle brand — see
|
|
25
|
+
/** Cross-bundle brand — see {@link errorKind}. Non-enumerable-ish (symbol key ⇒ not in JSON). */
|
|
26
26
|
readonly [ERROR_BRAND]: true;
|
|
27
|
+
/** Builds the error from `message`, and sets `name` to the class's name as written. */
|
|
27
28
|
constructor(message: string);
|
|
28
29
|
}
|
|
29
30
|
/**
|
|
@@ -33,6 +34,8 @@ export declare class CloudRoaringError extends Error {
|
|
|
33
34
|
*/
|
|
34
35
|
export declare class ValidationError extends CloudRoaringError {
|
|
35
36
|
}
|
|
37
|
+
/** What a contention {@link WriteConflictError} ends with: the call is safe to run again. */
|
|
38
|
+
export declare const CONTENTION_NEXT_STEP = "; another writer changed the segment's row while this call ran. Re-run the call: it reads the row afresh";
|
|
36
39
|
/** An OCC conditional write/delete lost the race — the row changed since it was read. */
|
|
37
40
|
export declare class WriteConflictError extends CloudRoaringError {
|
|
38
41
|
}
|
|
@@ -57,6 +60,7 @@ export declare class LeaseExpiredError extends CloudRoaringError {
|
|
|
57
60
|
readonly until: number;
|
|
58
61
|
/** `'expired'` when the clock reached `until`, `'released'` after `release()`. */
|
|
59
62
|
readonly reason: 'expired' | 'released';
|
|
63
|
+
/** Builds the error from its `message`, the epoch-ms the lease ran until, and why it ended. */
|
|
60
64
|
constructor(message: string, until: number, reason: 'expired' | 'released');
|
|
61
65
|
}
|
|
62
66
|
/**
|
|
@@ -73,11 +77,13 @@ export declare class LeaseLimitError extends CloudRoaringError {
|
|
|
73
77
|
* `'stale-operand'`. Deterministic: run the call again against the current operand.
|
|
74
78
|
*/
|
|
75
79
|
export declare class StaleOperandError extends CloudRoaringError {
|
|
80
|
+
/** The stable code `'stale-operand'`, to match on. */
|
|
76
81
|
readonly code: "stale-operand";
|
|
77
82
|
/** The operand's name in the call that found it stale. */
|
|
78
83
|
readonly operand: string;
|
|
79
84
|
/** Why it is stale. */
|
|
80
85
|
readonly reason: 'moved' | 'erased';
|
|
86
|
+
/** Builds the error from its `message`, the operand's name and why it is stale. */
|
|
81
87
|
constructor(message: string, operand: string, reason: 'moved' | 'erased');
|
|
82
88
|
}
|
|
83
89
|
/**
|
|
@@ -97,14 +103,47 @@ export declare class UnsupportedError extends CloudRoaringError {
|
|
|
97
103
|
export declare class CapabilityError extends CloudRoaringError {
|
|
98
104
|
}
|
|
99
105
|
/**
|
|
100
|
-
* An operation would exceed its per-op **denial-of-wallet
|
|
101
|
-
* `
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
+
* An operation would exceed one of its per-op **denial-of-wallet bounds** — too many backend requests
|
|
107
|
+
* (`budget.maxRequests`), too many bytes held at once by a combine (`maxBufferedBytes`), or too many registry rows
|
|
108
|
+
* in a scan (`maxScanSegments`) — so it is refused, before the fan-out where it can be known beforehand (hard
|
|
109
|
+
* invariant 6). Default-on but generous (normal ops never hit it); tune each per store or per op, or disable the
|
|
110
|
+
* request budget with `budget: false`. Deterministic (never retried): the op is too big by policy, not by luck.
|
|
111
|
+
* Each request's bytes are separately capped (the safe-deserialize ceiling), so bounding the request count
|
|
112
|
+
* transitively bounds bytes.
|
|
113
|
+
*
|
|
114
|
+
* Carries an option name and two numbers, and no data: `option` names the option to raise, `limit` is the bound in
|
|
115
|
+
* force, and `spent` is what the call had reached when it was refused. What `spent` measures depends on `option`:
|
|
116
|
+
*
|
|
117
|
+
* - `'budget.maxRequests'`: the requests the call needs, which raising the option to it lets through. For a plan, it
|
|
118
|
+
* is the planned reads (with the reads already taken, when a run plans again); an enumeration that
|
|
119
|
+
* stops at the bound reports `limit + 1`, a lower bound.
|
|
120
|
+
* - `'maxBufferedBytes'`: a **lower bound** on the bytes the call needs resident: what it held plus what it asked
|
|
121
|
+
* for at the moment it was refused. Raising the limit to `spent` can be refused again, further on, because the
|
|
122
|
+
* call had not finished growing. The one figure that lets an output through is the per-output refusal's, which
|
|
123
|
+
* the message gives as "raise maxBufferedBytes to at least N". Otherwise raise the limit with room to spare, or
|
|
124
|
+
* split the outputs across calls.
|
|
125
|
+
* - `'maxScanSegments'`: `limit + 1`, a lower bound: the scan stops at the first row past the bound (for the retention
|
|
126
|
+
* sweep's due-index scan, the first due-index pointer, unread, which may be stale), so it never learns the fleet's
|
|
127
|
+
* size.
|
|
128
|
+
*
|
|
129
|
+
* The message states the same limit.
|
|
106
130
|
*/
|
|
107
131
|
export declare class BudgetExceededError extends CloudRoaringError {
|
|
132
|
+
/** The option to raise to let the call through. */
|
|
133
|
+
readonly option: 'budget.maxRequests' | 'maxBufferedBytes' | 'maxScanSegments';
|
|
134
|
+
/** The bound in force: requests, bytes or rows, by `option`. */
|
|
135
|
+
readonly limit: number;
|
|
136
|
+
/**
|
|
137
|
+
* What the call had reached when refused. By `option`: the requests the call needs (`'budget.maxRequests'`), a
|
|
138
|
+
* lower bound on the resident bytes it needs (`'maxBufferedBytes'`), or `limit + 1` (`'maxScanSegments'`).
|
|
139
|
+
*/
|
|
140
|
+
readonly spent: number;
|
|
141
|
+
/** Builds the error from its `message` and the `option`, `limit` and `spent` it reports. */
|
|
142
|
+
constructor(message: string, details: {
|
|
143
|
+
readonly option: 'budget.maxRequests' | 'maxBufferedBytes' | 'maxScanSegments';
|
|
144
|
+
readonly limit: number;
|
|
145
|
+
readonly spent: number;
|
|
146
|
+
});
|
|
108
147
|
}
|
|
109
148
|
/**
|
|
110
149
|
* An encrypted segment's data key (DEK) cannot be unwrapped because the keystore holds none of the
|
|
@@ -132,6 +171,7 @@ export declare class KeyUnavailableError extends CloudRoaringError {
|
|
|
132
171
|
export declare class TransientError extends CloudRoaringError {
|
|
133
172
|
/** A second brand so a transient fault is classifiable cross-bundle. */
|
|
134
173
|
readonly [TRANSIENT_BRAND]: true;
|
|
174
|
+
/** Builds the error from `message`; `options.cause`, when given, is kept as `cause`. */
|
|
135
175
|
constructor(message: string, options?: {
|
|
136
176
|
cause?: unknown;
|
|
137
177
|
});
|
|
@@ -140,6 +180,30 @@ export declare class TransientError extends CloudRoaringError {
|
|
|
140
180
|
export declare function isCloudRoaringError(err: unknown): err is CloudRoaringError;
|
|
141
181
|
/** An OCC conditional write/delete lost the race — retry the read-modify-write, don't fail. */
|
|
142
182
|
export declare function isWriteConflictError(err: unknown): err is WriteConflictError;
|
|
183
|
+
/**
|
|
184
|
+
* A {@link WriteConflictError} for a registry `create` or `compareAndSwap` that may have landed before it met the
|
|
185
|
+
* conflict, so the conflict does not prove the write was refused. Throw it for every conflict of a write your backend's
|
|
186
|
+
* client may have sent more than once and that you cannot prove did not land: the read-back finds another writer's
|
|
187
|
+
* write over the row (a row is overwritten, so that writer may have swapped in over this write after it landed), or
|
|
188
|
+
* finds the row gone, or the compare-and-swap itself is answered that the row is gone. Throw a plain
|
|
189
|
+
* `WriteConflictError` for a conflict that does prove it: a write sent once, or a check your driver made before sending
|
|
190
|
+
* anything. A registry that wraps another rethrows the conflict it was given, not a new one, which would lose the mark.
|
|
191
|
+
*
|
|
192
|
+
* It is a `WriteConflictError` in every other way: `errorKind` names it `'WriteConflictError'`, `instanceof` holds on
|
|
193
|
+
* the same copy of the package, and every caller that handles a conflict handles it. A load's publish reads the row
|
|
194
|
+
* again, as after any conflict, and takes the write as its own when the row shows it there. Otherwise the write is
|
|
195
|
+
* reported to the load, as a write with no answer is, so a load refused after it carries `unanswered: true` on its
|
|
196
|
+
* `segment.load-refused` audit event. The mark is a registered `Symbol.for` brand, read
|
|
197
|
+
* across copies of the package. It is lost when the error is cloned or copied into a new one, and a conflict raised
|
|
198
|
+
* anywhere but a registry write is read as the plain conflict it also is.
|
|
199
|
+
*/
|
|
200
|
+
export declare function unsettledConflict(message: string): WriteConflictError;
|
|
201
|
+
/**
|
|
202
|
+
* Whether `err` is a conflict a driver raised with {@link unsettledConflict}: a {@link WriteConflictError}, by brand
|
|
203
|
+
* and name as {@link isWriteConflictError} reads it, that carries the unsettled mark. It reads the registered brand,
|
|
204
|
+
* so it holds for an error from another copy of the package. Like the other predicates, no entry exports it.
|
|
205
|
+
*/
|
|
206
|
+
export declare function isUnsettledConflict(err: unknown): boolean;
|
|
143
207
|
/** A retryable transient infrastructure fault. The retry layer keys on this. */
|
|
144
208
|
export declare function isTransientError(err: unknown): err is TransientError;
|
|
145
209
|
/** A read came after its pin's lease ended. */
|
|
@@ -164,4 +228,14 @@ export declare function isCapabilityError(err: unknown): err is CapabilityError;
|
|
|
164
228
|
export declare function isBudgetExceededError(err: unknown): err is BudgetExceededError;
|
|
165
229
|
/** No key the keystore holds can unwrap an encrypted segment's data key. */
|
|
166
230
|
export declare function isKeyUnavailableError(err: unknown): err is KeyUnavailableError;
|
|
231
|
+
/** The name of one of the library's error classes, as `errorKind` returns it. */
|
|
232
|
+
export type ErrorKind = 'CloudRoaringError' | 'ValidationError' | 'WriteConflictError' | 'IntegrityError' | 'NotFoundError' | 'UnsupportedError' | 'CapabilityError' | 'TransientError' | 'KeyUnavailableError' | 'BudgetExceededError' | 'LeaseExpiredError' | 'LeaseLimitError' | 'StaleOperandError';
|
|
233
|
+
/**
|
|
234
|
+
* Which of the library's error classes `err` is, or `undefined` when it is none of them. Works on every copy of
|
|
235
|
+
* the package, every bundle and every realm, where `instanceof` can silently stop matching: it reads the
|
|
236
|
+
* `Symbol.for` brands and the error's `name`. An application's own subclass of one class returns
|
|
237
|
+
* `'CloudRoaringError'`, except a subclass of `TransientError`, which returns `'TransientError'`; an error
|
|
238
|
+
* cloned out of a worker (the brand is gone) returns `undefined`.
|
|
239
|
+
*/
|
|
240
|
+
export declare function errorKind(err: unknown): ErrorKind | undefined;
|
|
167
241
|
export {};
|
package/dist/core/leases.d.ts
CHANGED
|
@@ -40,9 +40,11 @@ export declare function liveLeases(record: RegistryRecord | null, now: number):
|
|
|
40
40
|
* Both are optional; without a sleep a retry follows at once.
|
|
41
41
|
*/
|
|
42
42
|
export interface ChurnDeps {
|
|
43
|
+
/** The clock whose `sleep` waits out lease churn. Without a `sleep`, a retry follows at once. */
|
|
43
44
|
readonly clock?: {
|
|
44
45
|
readonly sleep?: (ms: number) => Promise<void>;
|
|
45
46
|
} | undefined;
|
|
47
|
+
/** The source that spreads each wait over a share of its bound. Without one, each wait is its whole bound. */
|
|
46
48
|
readonly rng?: Rng | undefined;
|
|
47
49
|
}
|
|
48
50
|
/**
|
|
@@ -17,7 +17,12 @@ import type { CombinedChunk } from './engine.js';
|
|
|
17
17
|
* (`getSerializationSizeInBytes`) is asked first, so one over the cap is refused before it is serialized.
|
|
18
18
|
*/
|
|
19
19
|
export interface PortableBitmap {
|
|
20
|
+
/** The bitmap in the portable Roaring format. */
|
|
20
21
|
serialize(format: 'portable'): Uint8Array;
|
|
22
|
+
/**
|
|
23
|
+
* The size in bytes of `serialize('portable')`, when the bitmap can say without serializing. A load asks it first, so
|
|
24
|
+
* an input over the size cap is refused before it is serialized.
|
|
25
|
+
*/
|
|
21
26
|
getSerializationSizeInBytes?(format: 'portable'): number;
|
|
22
27
|
}
|
|
23
28
|
/**
|
|
@@ -53,6 +58,49 @@ export declare class ChunkLoadInput {
|
|
|
53
58
|
readonly chunks: AsyncIterable<CombinedChunk>;
|
|
54
59
|
constructor(chunks: AsyncIterable<CombinedChunk>);
|
|
55
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Mark `err` as raised by a load's input (its ids, a bitmap's encoding, a combine's chunks) rather than by the write,
|
|
63
|
+
* and return it. A write-once conflict is a lost race only when the write raised it: a source that reads another
|
|
64
|
+
* segment (through a pin, say) can throw this library's own `WriteConflictError`, which is the source's to report.
|
|
65
|
+
*/
|
|
66
|
+
export declare function raisedBySource<T>(err: T): T;
|
|
67
|
+
/** Whether a load's input raised `err` ({@link raisedBySource}). */
|
|
68
|
+
export declare function isSourceError(err: unknown): boolean;
|
|
69
|
+
/**
|
|
70
|
+
* Ids the caller says ascend (`sorted: true`), written chunk by chunk while they are read, during the object's upload,
|
|
71
|
+
* so a load holds one chunk and the writer's index rather than the whole generation. Made only by
|
|
72
|
+
* {@link sortedLoadInput}, from ids.
|
|
73
|
+
*
|
|
74
|
+
* The first error the ids raise, or their check raises (an id out of range, or lower than the one before it), or the
|
|
75
|
+
* chunker raises building a chunk, is kept here, and the load throws that same error. The driver's upload never sees
|
|
76
|
+
* it: the chunker throws a sentinel of its own into the writer instead, so no driver maps, redacts or probes the
|
|
77
|
+
* caller's error on its way out.
|
|
78
|
+
*/
|
|
79
|
+
export declare class SortedLoadInput {
|
|
80
|
+
readonly ids: Iterable<number> | AsyncIterable<number>;
|
|
81
|
+
private failure;
|
|
82
|
+
constructor(ids: Iterable<number> | AsyncIterable<number>);
|
|
83
|
+
/** Keep `error` as what the ids raised, unless an earlier one is kept. */
|
|
84
|
+
fail(error: unknown): void;
|
|
85
|
+
/** What the load throws for a write that failed with `err`: the error the ids raised, when they raised one. */
|
|
86
|
+
thrown(err: unknown): unknown;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Whether a load was asked for `sorted: true`. `undefined` and `false` mean no; anything else that is not a boolean is
|
|
90
|
+
* refused before any request, since `sorted: 'yes'` read as absent would hold the whole generation in memory instead.
|
|
91
|
+
*
|
|
92
|
+
* @throws {ValidationError} for a value that is not a boolean.
|
|
93
|
+
*/
|
|
94
|
+
export declare function wantsSorted(sorted: unknown): boolean;
|
|
95
|
+
/**
|
|
96
|
+
* The input a load reads under its `sorted` option: ids as {@link SortedLoadInput} when it is `true`, anything else
|
|
97
|
+
* unchanged.
|
|
98
|
+
*
|
|
99
|
+
* @throws {ValidationError} for a `sorted` that is not a boolean, and for `sorted: true` with `{ serialized }`,
|
|
100
|
+
* `{ bitmap }` or a bitmap of the flavor's own: those are written from their own chunks already, so the option would
|
|
101
|
+
* do nothing, and an option that does nothing is refused rather than ignored.
|
|
102
|
+
*/
|
|
103
|
+
export declare function sortedLoadInput(prepared: ReturnType<typeof prepareLoadInput>, sorted: unknown): ReturnType<typeof prepareLoadInput> | SortedLoadInput;
|
|
56
104
|
/**
|
|
57
105
|
* Check a load's input and decode a bitmap input, before the load makes any request.
|
|
58
106
|
*
|