@cloudbitmaps/core 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/dist/chunk-7GT632UF.js +641 -0
- package/dist/chunk-7GT632UF.js.map +7 -0
- package/dist/chunk-CGILDXGP.js +73 -0
- package/dist/chunk-CGILDXGP.js.map +7 -0
- package/dist/chunk-GKHERVVQ.js +121 -0
- package/dist/chunk-GKHERVVQ.js.map +7 -0
- package/dist/{chunk-OSU7W2Y3.js → chunk-MVUD4R72.js} +200 -756
- package/dist/chunk-MVUD4R72.js.map +7 -0
- package/dist/core/audit.d.ts +53 -15
- package/dist/core/bit-route.d.ts +2 -0
- package/dist/core/blob.d.ts +4 -0
- package/dist/core/budget.d.ts +8 -3
- package/dist/core/check.d.ts +49 -0
- package/dist/core/combine-many.d.ts +11 -0
- package/dist/core/consistency.d.ts +11 -2
- package/dist/core/cooperative.d.ts +7 -0
- package/dist/core/crbm-storage-source.d.ts +7 -4
- package/dist/core/crypto.d.ts +5 -0
- package/dist/core/discover.d.ts +5 -3
- package/dist/core/engine.d.ts +4 -4
- package/dist/core/erase-id.d.ts +35 -12
- package/dist/core/erasure.d.ts +61 -27
- package/dist/core/errors.d.ts +30 -0
- package/dist/core/leases.d.ts +2 -0
- package/dist/core/load-input.d.ts +48 -0
- package/dist/core/load.d.ts +23 -2
- package/dist/core/metrics.d.ts +21 -0
- package/dist/core/pinned-storage-source.d.ts +27 -8
- package/dist/core/pointer-id.d.ts +9 -0
- package/dist/core/ports.d.ts +100 -7
- package/dist/core/preflight.d.ts +80 -0
- package/dist/core/retention-sweep.d.ts +60 -15
- package/dist/core/retention.d.ts +17 -6
- package/dist/core/retry.d.ts +1 -2
- package/dist/core/rollback.d.ts +22 -12
- package/dist/core/validate.d.ts +33 -2
- package/dist/driver-kit.d.ts +3 -3
- package/dist/driver-kit.js +61 -20
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +24 -1
- package/dist/drivers/_shared/options.d.ts +15 -7
- package/dist/drivers/backends.d.ts +17 -2
- package/dist/drivers/crypto.d.ts +17 -2
- package/dist/export/index.d.ts +16 -2
- package/dist/index.d.ts +10 -9
- package/dist/index.js +1743 -1701
- package/dist/index.js.map +4 -4
- package/dist/node.d.ts +14 -0
- package/dist/node.js +1064 -0
- package/dist/node.js.map +7 -0
- package/package.json +11 -2
- package/dist/chunk-OSU7W2Y3.js.map +0 -7
package/dist/core/audit.d.ts
CHANGED
|
@@ -73,10 +73,13 @@ export type AuditEvent = {
|
|
|
73
73
|
* segment's registry row changed while the load was writing). `cardinality` is what the refused generation
|
|
74
74
|
* would have contained, and `0` for a load that lost its generation number and wrote nothing.
|
|
75
75
|
*
|
|
76
|
-
* `unanswered` is present, and `true`, on a `'superseded'` refusal
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
76
|
+
* `unanswered` is present, and `true`, on a `'superseded'` refusal after a registry write whose answer did not
|
|
77
|
+
* say whether it landed: one that ended without an answer, or a conflict its registry marked as one that may
|
|
78
|
+
* follow the write's own landing (Azure Blob marks every conflict of a row write, since its client may have sent
|
|
79
|
+
* the write twice). That write may have landed, so the generation may have been the segment's current one for a
|
|
80
|
+
* while before another write replaced it, or, where a write renewed the row's `pointerId` over it, may be current
|
|
81
|
+
* still. Absent when every write was answered and every conflict was one its registry vouches the write did not
|
|
82
|
+
* land.
|
|
80
83
|
*/
|
|
81
84
|
readonly kind: 'segment.load-refused';
|
|
82
85
|
readonly namespace?: string;
|
|
@@ -115,11 +118,12 @@ export type AuditEvent = {
|
|
|
115
118
|
* not hold, or an encrypted one under a cleartext row (a first load's that lost the race, or crashed). The one
|
|
116
119
|
* emitter is `eraseIdFromSegment`, once a listing of the bucket shows nothing left holding the id, so the record
|
|
117
120
|
* exists only for a finished erasure. A call that ends otherwise throws or reports `erased: false` with another
|
|
118
|
-
* reason and emits nothing, even when it had deleted a holder; its re-run emits when it finishes, and
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
* segment's pointer on a generation it had
|
|
122
|
-
* the record although the call cannot
|
|
121
|
+
* reason and emits nothing, even when it had deleted a holder; its re-run emits when it finishes, and, when
|
|
122
|
+
* another collection deleted the last holder meanwhile, emits nothing and answers `'not-member'`, which
|
|
123
|
+
* `eraseSubject`'s ledger shows by listing nothing for the segment. One finished erasure emits it and then
|
|
124
|
+
* throws `NotFoundError`: one that found, after its deletes, the segment's pointer on a generation it had
|
|
125
|
+
* deleted (a rollback moved it there meanwhile), so the deletion is on the record although the call cannot
|
|
126
|
+
* answer.
|
|
123
127
|
*
|
|
124
128
|
* `fromGeneration` is the newest generation the call found holding the id, and is absent when it found the id in
|
|
125
129
|
* none and deleted only objects it could not search: then the ledger lists no entry for the segment, and this
|
|
@@ -137,24 +141,52 @@ export type AuditEvent = {
|
|
|
137
141
|
readonly collected: readonly number[];
|
|
138
142
|
} | {
|
|
139
143
|
/**
|
|
140
|
-
* A segment was **crypto-shredded
|
|
141
|
-
*
|
|
142
|
-
*
|
|
144
|
+
* A segment was **crypto-shredded**: its wrapped DEK(s) are gone from the segment's **current** registry row,
|
|
145
|
+
* so nothing that opens the segment through that row can decrypt its Storage bytes, in the bucket or in any
|
|
146
|
+
* backup, replica or WORM copy of its objects. It destroys no KEK: a noncurrent version, a backup or a
|
|
147
|
+
* point-in-time copy of the row still holds the wrapped DEK(s), and a registry restore to a point before the
|
|
148
|
+
* shred makes the segment readable again wherever its objects survive. The destruction is complete once no
|
|
149
|
+
* retained copy of the row holds them, or once every KEK that wrapped them is destroyed. A process that had
|
|
150
|
+
* already opened the segment keeps its unwrapped key in memory until told (`store.invalidate(ref)`). Emitted
|
|
151
|
+
* only for a genuine key shred, never for a cleartext tombstone (which leaves the Storage bytes readable) or
|
|
152
|
+
* an idempotent re-run.
|
|
143
153
|
*/
|
|
144
154
|
readonly kind: 'segment.erase';
|
|
145
155
|
readonly namespace?: string;
|
|
146
156
|
readonly segment: string;
|
|
147
157
|
/** The incarnation of the row the shred turned into a tombstone. */
|
|
148
158
|
readonly incarnation?: string;
|
|
159
|
+
} | {
|
|
160
|
+
/**
|
|
161
|
+
* A segment was tombstoned **without a key shred**: `destroySegment` or `eraseNamespace` with `allowCleartext`
|
|
162
|
+
* turned the row of a segment that holds no wrapped key into a `destroyed` tombstone. No read resolves the
|
|
163
|
+
* segment any more, but nothing was deleted and no key was destroyed, so its objects stay readable in the
|
|
164
|
+
* bucket, in any backup and through any copy of the row from before the tombstone.
|
|
165
|
+
*
|
|
166
|
+
* Deliberately not {@link AuditEvent} `segment.erase`, which attests a key shred, nor `segment.dispose`, which
|
|
167
|
+
* attests that a drop swept the storage: this one is the record that the row was tombstoned, which is what a
|
|
168
|
+
* registry restore undoes and what the restore procedure looks for. `objectsReadable` is always `true`; it
|
|
169
|
+
* is a field so a consumer that forwards events cannot read this one as an erasure receipt. Emitted once per
|
|
170
|
+
* tombstone this call wrote, not for the idempotent re-run, and not for a segment that was encrypted
|
|
171
|
+
* (`segment.erase`), or dropped (`segment.dispose`).
|
|
172
|
+
*/
|
|
173
|
+
readonly kind: 'segment.tombstone';
|
|
174
|
+
readonly namespace?: string;
|
|
175
|
+
readonly segment: string;
|
|
176
|
+
/** The incarnation of the row the call turned into a tombstone. */
|
|
177
|
+
readonly incarnation?: string;
|
|
178
|
+
/** Always `true`: the segment's objects were not deleted and no key was destroyed. */
|
|
179
|
+
readonly objectsReadable: true;
|
|
149
180
|
} | {
|
|
150
181
|
/**
|
|
151
182
|
* A segment was **disposed of** — tombstoned and its storage reclaimed (its Storage generations deleted) by
|
|
152
183
|
* `dropSegment`, *without* a key shred.
|
|
153
184
|
*
|
|
154
185
|
* Deliberately a separate kind from {@link AuditEvent} `segment.erase`, and the distinction is the point.
|
|
155
|
-
* `segment.erase` attests that
|
|
156
|
-
*
|
|
157
|
-
*
|
|
186
|
+
* `segment.erase` attests that the current row no longer holds the key, which is the one erasure claim that
|
|
187
|
+
* reaches WORM copies of the objects, once no retained copy of the row holds it. Deleting an object is
|
|
188
|
+
* strictly weaker: a noncurrent version, a cross-region replica or a PITR snapshot can still hold the
|
|
189
|
+
* cleartext. Emitting one kind for both would make a compliance dashboard
|
|
158
190
|
* over-attest, so a cleartext drop gets this instead.
|
|
159
191
|
*
|
|
160
192
|
* An **encrypted** segment dropped via `dropSegment` emits **both** — `segment.erase` for the key shred and
|
|
@@ -178,6 +210,10 @@ export type AuditEvent = {
|
|
|
178
210
|
};
|
|
179
211
|
/** Sink for {@link AuditEvent}s. Injected via the lifecycle options; omit it and nothing is recorded. */
|
|
180
212
|
export interface IAuditSink {
|
|
213
|
+
/**
|
|
214
|
+
* Receives one audit event, called synchronously by the operation that caused it. What it throws, or rejects with, is
|
|
215
|
+
* swallowed and never breaks that operation.
|
|
216
|
+
*/
|
|
181
217
|
onEvent(event: AuditEvent): void;
|
|
182
218
|
}
|
|
183
219
|
/** The default sink: records nothing. */
|
|
@@ -200,8 +236,10 @@ export declare function ignoreRejection(returned: unknown): void;
|
|
|
200
236
|
/** A ready-made sink that records events into an in-memory list — handy for tests + simple audit trails. */
|
|
201
237
|
export declare class RecordingAuditSink implements IAuditSink {
|
|
202
238
|
private readonly recorded;
|
|
239
|
+
/** Records the event, in emission order. */
|
|
203
240
|
onEvent(event: AuditEvent): void;
|
|
204
241
|
/** An independent copy of the recorded events, in emission order. */
|
|
205
242
|
snapshot(): AuditEvent[];
|
|
243
|
+
/** Forgets every recorded event. */
|
|
206
244
|
reset(): void;
|
|
207
245
|
}
|
package/dist/core/bit-route.d.ts
CHANGED
|
@@ -13,6 +13,8 @@ interface BitRoute {
|
|
|
13
13
|
}
|
|
14
14
|
/** Was `err` thrown by {@link splitId} for an id outside the u32 range? */
|
|
15
15
|
export declare function isBadIdError(err: unknown): err is ValidationError;
|
|
16
|
+
/** The refusal {@link splitId} throws for a value that is not a u32 id, for a caller that checks the range itself. */
|
|
17
|
+
export declare function refusalOfId(id: unknown): ValidationError;
|
|
16
18
|
/** Split a u32 id into `{ chunkKey, remainder }`. Throws `ValidationError` for non-u32 input. */
|
|
17
19
|
export declare function splitId(id: number): BitRoute;
|
|
18
20
|
/**
|
package/dist/core/blob.d.ts
CHANGED
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
* the write resolves, so a sink that holds bytes past that copies them.
|
|
4
4
|
*/
|
|
5
5
|
export interface BlobSink {
|
|
6
|
+
/**
|
|
7
|
+
* Appends `bytes` to the object. Once it resolves the caller may reuse the buffer, so a sink that keeps the bytes
|
|
8
|
+
* copies them.
|
|
9
|
+
*/
|
|
6
10
|
write(bytes: Uint8Array): Promise<void>;
|
|
7
11
|
}
|
|
8
12
|
/**
|
package/dist/core/budget.d.ts
CHANGED
|
@@ -15,6 +15,11 @@
|
|
|
15
15
|
* search generation by generation is also charged one unit for each generation it opens beyond the current one,
|
|
16
16
|
* since that is O(generations) work the segment count alone does not show.
|
|
17
17
|
*/
|
|
18
|
+
/**
|
|
19
|
+
* The cap on one operation's fan-out: its `maxRequests` units, counted in storage chunk fetches for a read and in
|
|
20
|
+
* segments scanned for an admin scan. `eraseSubject` also charges one unit for each generation beyond the current one
|
|
21
|
+
* that it opens, in a segment it has to search generation by generation.
|
|
22
|
+
*/
|
|
18
23
|
export interface Budget {
|
|
19
24
|
/** Max units of fan-out (Storage chunk fetches, or segments scanned) one op may issue before it is refused. */
|
|
20
25
|
readonly maxRequests: number;
|
|
@@ -30,13 +35,13 @@ export type BudgetOption = Partial<Budget> | false;
|
|
|
30
35
|
/**
|
|
31
36
|
* Resolve a budget option against a fallback to a concrete `Budget`, or `null` when disabled. Validates: anything but
|
|
32
37
|
* `false`, an absent value or an object of `{ maxRequests }` is refused, since a cap of the wrong shape (`5`,
|
|
33
|
-
* `{ maxRequest: 5 }`) would otherwise read as no override and leave the fallback in force.
|
|
34
|
-
*
|
|
38
|
+
* `{ maxRequest: 5 }`) would otherwise read as no override and leave the fallback in force. A key whose value is `undefined` reads
|
|
39
|
+
* as absent; a `null` budget and a `maxRequests` of `null` are refused.
|
|
35
40
|
*/
|
|
36
41
|
export declare function resolveBudget(opt: BudgetOption | undefined, fallback: Budget): Budget | null;
|
|
37
42
|
/**
|
|
38
43
|
* Resolve a **per-op** budget override against the store's own resolved budget (never the raw global default):
|
|
39
|
-
* `undefined`
|
|
44
|
+
* `undefined` inherits the store budget as-is (including a disabled `null`); a partial `{}` / omitted
|
|
40
45
|
* `maxRequests` inherits the store's *tightening* (so a per-op passthrough of an absent config value can't
|
|
41
46
|
* silently lift a tight tenant ceiling back to the generous default); `{ maxRequests }` replaces it; `false`
|
|
42
47
|
* disables. Falls back to {@link DEFAULT_BUDGET} only when the store itself has no budget (disabled).
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A value as a message shows it: a number or boolean as itself, a bigint with an `n` (so `5n` is not mistaken for
|
|
3
|
+
* `5`), a string quoted (so `'200'` is not mistaken for `200`), and anything else by its type.
|
|
4
|
+
*/
|
|
5
|
+
export declare function shown(value: unknown): string;
|
|
6
|
+
/**
|
|
7
|
+
* `value` if it is a safe integer from `min` to `max` (inclusive; `max` defaults to the largest safe integer), else a
|
|
8
|
+
* `ValidationError` naming `what`: `<what> must be an integer from <min> to <max>; got <value>`.
|
|
9
|
+
*/
|
|
10
|
+
export declare function requireInteger(value: unknown, bounds: {
|
|
11
|
+
readonly min: number;
|
|
12
|
+
readonly max?: number;
|
|
13
|
+
readonly what: string;
|
|
14
|
+
readonly note?: string;
|
|
15
|
+
}): number;
|
|
16
|
+
/**
|
|
17
|
+
* Refuse a `requireEncryption` that is present and not exactly a boolean. Every site that acts on the flag compares it
|
|
18
|
+
* with `=== true`, so a truthy string would be taken for `true` by one and for `false` by another.
|
|
19
|
+
*/
|
|
20
|
+
export declare function checkRequireEncryption(value: unknown): void;
|
|
21
|
+
/** `value` if it is a safe integer of 1 or more, else a `ValidationError` naming `what`. */
|
|
22
|
+
export declare function requirePositiveInteger(value: unknown, what: string): number;
|
|
23
|
+
/**
|
|
24
|
+
* `value` if it is a finite number within the optional `min` and `max` (inclusive), else a `ValidationError` naming
|
|
25
|
+
* `what`: `<what> must be a finite number from <min> to <max>; got <value>`. `NaN` and the infinities are refused.
|
|
26
|
+
*/
|
|
27
|
+
export declare function requireFiniteNumber(value: unknown, bounds: {
|
|
28
|
+
readonly min?: number;
|
|
29
|
+
readonly max?: number;
|
|
30
|
+
readonly what: string;
|
|
31
|
+
readonly note?: string;
|
|
32
|
+
}): number;
|
|
33
|
+
/**
|
|
34
|
+
* Refuse an options bag that is not an object, or that holds a key not in `keys`, naming each such key and the keys it
|
|
35
|
+
* takes. `hint`, when given, ends the message, to point at where a likely mistake belongs (`an S3 client goes in
|
|
36
|
+
* \`client\``). Only the bag's own enumerable string keys are read.
|
|
37
|
+
*/
|
|
38
|
+
export declare function refuseUnknownOptions(name: string, options: unknown, keys: readonly string[], hint?: string): void;
|
|
39
|
+
/**
|
|
40
|
+
* The option `name`'s `value`, refused with `ValidationError` unless it is absent or a boolean: a truthy string or a
|
|
41
|
+
* `1` is not taken for `true`, and `null` is not taken for absent. An absent value comes back `undefined`, for the
|
|
42
|
+
* caller's default.
|
|
43
|
+
*/
|
|
44
|
+
export declare function requireBool(name: string, value: unknown): boolean | undefined;
|
|
45
|
+
/**
|
|
46
|
+
* The option `name`'s `value`, refused with `ValidationError` unless it is absent or a string with something in it
|
|
47
|
+
* (one that is empty or only whitespace is refused, as is `null`). An absent value comes back `undefined`.
|
|
48
|
+
*/
|
|
49
|
+
export declare function requireNonEmptyString(name: string, value: unknown): string | undefined;
|
|
@@ -163,6 +163,10 @@ export interface MaterializeManyOutputStats {
|
|
|
163
163
|
readonly group: number | null;
|
|
164
164
|
/** Clock time the group's pass began and the output's publish settled (or it failed), when it ran. */
|
|
165
165
|
readonly startedAt: number | null;
|
|
166
|
+
/**
|
|
167
|
+
* Clock time (ms) when its publish settled or failed, or `null` when the output never ran. Read from the same clock
|
|
168
|
+
* as `startedAt`.
|
|
169
|
+
*/
|
|
166
170
|
readonly endedAt: number | null;
|
|
167
171
|
/** Chunks the output holds. */
|
|
168
172
|
readonly chunks: number;
|
|
@@ -178,6 +182,7 @@ export interface MaterializeManyOutputStats {
|
|
|
178
182
|
export interface MaterializeManyStats {
|
|
179
183
|
/** Groups run, the re-runs of outputs a lying index made too big included. */
|
|
180
184
|
readonly groups: number;
|
|
185
|
+
/** The requests the call made, by class. */
|
|
181
186
|
readonly requests: {
|
|
182
187
|
/** Range requests sent for operand chunks (retries of one request count once). */
|
|
183
188
|
readonly rangeReads: number;
|
|
@@ -209,6 +214,7 @@ export interface MaterializeManyStats {
|
|
|
209
214
|
readonly put: number;
|
|
210
215
|
};
|
|
211
216
|
};
|
|
217
|
+
/** The chunk-read budget: the limit in force, the reads the plan needed and the reads made. */
|
|
212
218
|
readonly budget: {
|
|
213
219
|
/** The limit used: `null` when lifted. */
|
|
214
220
|
readonly maxRequests: number | null;
|
|
@@ -217,6 +223,9 @@ export interface MaterializeManyStats {
|
|
|
217
223
|
/** Chunk reads made. */
|
|
218
224
|
readonly used: number;
|
|
219
225
|
};
|
|
226
|
+
/**
|
|
227
|
+
* The memory the call held: its limit, the most the ledger held at once and what the plan holds for the whole call.
|
|
228
|
+
*/
|
|
220
229
|
readonly memory: {
|
|
221
230
|
readonly maxBufferedBytes: number;
|
|
222
231
|
/** The most the ledger held at once. */
|
|
@@ -224,6 +233,7 @@ export interface MaterializeManyStats {
|
|
|
224
233
|
/** What the plan holds for the whole call, charged to the ledger: operand indexes, and each output's keys and expression. */
|
|
225
234
|
readonly planBytes: number;
|
|
226
235
|
};
|
|
236
|
+
/** How many chunk keys the call skipped reading. */
|
|
227
237
|
readonly chunks: {
|
|
228
238
|
/** Index keys not read because no output needed them there, summed over groups. */
|
|
229
239
|
readonly pruned: number;
|
|
@@ -238,6 +248,7 @@ export interface MaterializeManyStats {
|
|
|
238
248
|
};
|
|
239
249
|
/** By operand name, an object with no prototype, so any name is a key. */
|
|
240
250
|
readonly operands: Readonly<Record<string, MaterializeManyOperandStats>>;
|
|
251
|
+
/** One entry per output of the call, in the order the call gave them. */
|
|
241
252
|
readonly outputs: readonly MaterializeManyOutputStats[];
|
|
242
253
|
}
|
|
243
254
|
export interface CombineManyRun<R> {
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* the affected segment then throw. This scan
|
|
8
8
|
* detects it up front (run it at startup after a restore) instead of discovering it on the first read.
|
|
9
9
|
*
|
|
10
|
-
* Read-only; bounded fan-out. `destroyed`
|
|
11
|
-
*
|
|
10
|
+
* Read-only; bounded fan-out. `destroyed` segments are skipped: no generation resolves through one, so a missing
|
|
11
|
+
* object under it is not a torn restore. A segment whose Storage/registry can't be read this pass is recorded in
|
|
12
12
|
* `errored` (never aborts the scan). One segment can add several entries to `inconsistent` (its current object, and with
|
|
13
13
|
* `generations` one per known generation whose object is not the one recorded) and can also be in `errored`, when a
|
|
14
14
|
* read failed after those were found; with `generations` a segment costs up to one read per entry of its ledger.
|
|
@@ -35,8 +35,11 @@ import type { IKeystore } from './crypto.js';
|
|
|
35
35
|
* something a modest operator box survives. Raisable, because a ceiling you cannot lift is a landmine.
|
|
36
36
|
*/
|
|
37
37
|
export { DEFAULT_MAX_SCAN_SEGMENTS } from './registry-scan.js';
|
|
38
|
+
/** One problem the consistency check found with one segment; `issue` says which. */
|
|
38
39
|
export interface ConsistencyIssue {
|
|
40
|
+
/** The segment's name. */
|
|
39
41
|
readonly segment: string;
|
|
42
|
+
/** The segment's namespace, or `undefined` for the default one. */
|
|
40
43
|
readonly namespace?: string;
|
|
41
44
|
/**
|
|
42
45
|
* The registry's `currentGen` for the segment. For `missing-storage-generation` it is the generation whose `.crbm` is
|
|
@@ -70,11 +73,17 @@ export interface ConsistencyIssue {
|
|
|
70
73
|
}
|
|
71
74
|
/** A segment that could not be checked this pass (Storage/registry read fault) — not proof of a torn restore. */
|
|
72
75
|
export interface ConsistencyErrorEntry {
|
|
76
|
+
/** The name of the segment that could not be checked. */
|
|
73
77
|
readonly segment: string;
|
|
78
|
+
/** That segment's namespace, or `undefined` for the default one. */
|
|
74
79
|
readonly namespace?: string;
|
|
75
80
|
/** The read error's message (never the raw error, so the report stays serializable/loggable). */
|
|
76
81
|
readonly error: string;
|
|
77
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* What a consistency check found: how many segments it scanned, the issues, the segments it could not read, and, when
|
|
85
|
+
* asked for, the checks it could not make.
|
|
86
|
+
*/
|
|
78
87
|
export interface ConsistencyReport {
|
|
79
88
|
/** Registered segments scanned (a `destroyed` segment counts as scanned but is never an issue). */
|
|
80
89
|
readonly checked: number;
|
|
@@ -51,6 +51,13 @@ export interface Yielder {
|
|
|
51
51
|
* (~60 yields across a 1M-id load) while still capping any single blocking stretch in the low tens of ms.
|
|
52
52
|
*/
|
|
53
53
|
export declare const YIELD_EVERY = 1024;
|
|
54
|
+
/**
|
|
55
|
+
* Ids between a load's ingest-loop yields, sorted or not. 16x coarser than the per-chunk cadence because the per-id
|
|
56
|
+
* work is ~40 ns (a `splitId` and an array push) against ~1.3 µs per chunk — matching cadences would put the yield cost
|
|
57
|
+
* on the wrong side of the work it interrupts. Sized against the *slowest* ingest, an in-memory async generator at
|
|
58
|
+
* ~220 ns/id, so the resulting stretch stays a few ms there and well under 1 ms on a plain array.
|
|
59
|
+
*/
|
|
60
|
+
export declare const YIELD_EVERY_IDS: number;
|
|
54
61
|
/**
|
|
55
62
|
* Build a periodic yield.
|
|
56
63
|
*
|
|
@@ -377,7 +377,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
377
377
|
* the one read path with no resilience.
|
|
378
378
|
*
|
|
379
379
|
* **`status` is re-checked here**, on every open rather than once at pin time. A pin taken before a
|
|
380
|
-
* crypto-shred must not keep unwrapping a DEK the shred
|
|
380
|
+
* crypto-shred must not keep unwrapping a DEK whose wrapping the shred removed; a destroyed segment resolves no
|
|
381
|
+
* generation
|
|
381
382
|
* for anyone, and a pin's read of one fails. That holds from the pin's next open: a reader already open keeps
|
|
382
383
|
* the key it unwrapped until it is evicted or invalidated.
|
|
383
384
|
*
|
|
@@ -419,7 +420,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
419
420
|
/**
|
|
420
421
|
* Whether the segment is a registered name at all. With a registry: a row that is not a `destroyed` tombstone,
|
|
421
422
|
* including one with no generation yet (minted by `setRetention` before the first load), which somebody deliberately
|
|
422
|
-
* created.
|
|
423
|
+
* created. No generation resolves through a tombstone, so it answers `false`. Without a registry there is no
|
|
424
|
+
* registration to
|
|
423
425
|
* consult, so the honest answer is whether the bucket holds any generation of it.
|
|
424
426
|
*
|
|
425
427
|
* Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
|
|
@@ -569,8 +571,9 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
|
|
|
569
571
|
*
|
|
570
572
|
* A *second* NotFound is pathological (GC outrunning resolution) and propagates rather than fabricating an
|
|
571
573
|
* absent answer — never return a wrong result. `ifGone` is returned when the segment has no committed
|
|
572
|
-
* generation to serve: storage is empty, or the row
|
|
573
|
-
*
|
|
574
|
+
* generation to serve: storage is empty, or the row is destroyed (crypto-shredded, destroyed in the clear or
|
|
575
|
+
* dropped), in which case
|
|
576
|
+
* reading empty is the documented outcome rather than a failure.
|
|
574
577
|
*
|
|
575
578
|
* **Bounded to exactly two resolve-and-open round trips.** That is a cost contract, not a detail: the retry
|
|
576
579
|
* re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
|
package/dist/core/crypto.d.ts
CHANGED
|
@@ -8,8 +8,11 @@
|
|
|
8
8
|
*/
|
|
9
9
|
/** One AEAD result: a fresh per-message nonce, the ciphertext, and the authentication tag. */
|
|
10
10
|
export interface AeadSealed {
|
|
11
|
+
/** The fresh per-message nonce. */
|
|
11
12
|
readonly nonce: Uint8Array;
|
|
13
|
+
/** The encrypted bytes. */
|
|
12
14
|
readonly ciphertext: Uint8Array;
|
|
15
|
+
/** The authentication tag. */
|
|
13
16
|
readonly tag: Uint8Array;
|
|
14
17
|
}
|
|
15
18
|
/**
|
|
@@ -36,7 +39,9 @@ export interface Aead {
|
|
|
36
39
|
* plaintext key material — safe to store in the registry next to the (encrypted) data.
|
|
37
40
|
*/
|
|
38
41
|
export interface WrappedDek {
|
|
42
|
+
/** The id of the key-encryption key (KEK) this wrapping is under. */
|
|
39
43
|
readonly keyId: string;
|
|
44
|
+
/** The wrapped data key, an opaque base64 string. */
|
|
40
45
|
readonly wrapped: string;
|
|
41
46
|
}
|
|
42
47
|
/**
|
package/dist/core/discover.d.ts
CHANGED
|
@@ -20,9 +20,11 @@ export interface SegmentInfo extends SegmentRef {
|
|
|
20
20
|
*/
|
|
21
21
|
readonly currentGen: number | null;
|
|
22
22
|
/**
|
|
23
|
-
* `destroyed` is a tombstone: the row stays so sweeps can still find it, and
|
|
24
|
-
*
|
|
25
|
-
*
|
|
23
|
+
* `destroyed` is a tombstone: the row stays so sweeps can still find it, and no generation resolves through it. A
|
|
24
|
+
* `dropSegment` deletes the generations (check its `generationsRemaining`); a crypto-shred leaves them in the bucket
|
|
25
|
+
* and removes the DEK wrappings from the row, which makes every copy of the bytes unreadable once no retained copy of
|
|
26
|
+
* the row still holds them, or every KEK that wrapped them is destroyed. A cleartext segment destroyed with
|
|
27
|
+
* `allowCleartext` has no key to remove: its objects stay in the bucket and stay readable.
|
|
26
28
|
*/
|
|
27
29
|
readonly status: RegistryRecord['status'];
|
|
28
30
|
}
|
package/dist/core/engine.d.ts
CHANGED
|
@@ -346,11 +346,11 @@ export declare class SegmentEngine {
|
|
|
346
346
|
* for the range's edge, never a chunk the cache holds or another operand's. The reads, the budget, the metric and the
|
|
347
347
|
* errors are the id reads' own.
|
|
348
348
|
*/
|
|
349
|
-
intersectChunks(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<CombinedChunk>;
|
|
349
|
+
intersectChunks(segs: readonly SegmentRef[], options?: CombineOptions, label?: string): AsyncGenerator<CombinedChunk>;
|
|
350
350
|
/** {@link union}, as chunks: see {@link intersectChunks}. */
|
|
351
|
-
unionChunks(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<CombinedChunk>;
|
|
351
|
+
unionChunks(segs: readonly SegmentRef[], options?: CombineOptions, label?: string): AsyncGenerator<CombinedChunk>;
|
|
352
352
|
/** {@link andNot}, as chunks: see {@link intersectChunks}. */
|
|
353
|
-
andNotChunks(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'
|
|
353
|
+
andNotChunks(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>, label?: string): AsyncGenerator<CombinedChunk>;
|
|
354
354
|
/**
|
|
355
355
|
* Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
|
|
356
356
|
* absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
|
|
@@ -546,7 +546,7 @@ export declare class SegmentEngine {
|
|
|
546
546
|
* the read listed the segment's keys: the read then lists them again, and caches as a read begun after the call.
|
|
547
547
|
*
|
|
548
548
|
* The decoded-chunk cache is keyed by generation, which handles a *publish* (the new generation misses) but
|
|
549
|
-
* not a *destruction*: a segment that was erased from, dropped,
|
|
549
|
+
* not a *destruction*: a segment that was erased from, dropped, destroyed or retired has no newer generation
|
|
550
550
|
* whose key would miss, so its decoded chunks stay resident and keep answering. That is why the erasing
|
|
551
551
|
* process itself could report `erased: true` and then still answer `true` for the same id, out of RAM, with
|
|
552
552
|
* no storage read to intercept.
|
package/dist/core/erase-id.d.ts
CHANGED
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
* There is no per-id delete on an immutable object, and there is no mutable tier to hold a tombstone, so erasure
|
|
6
6
|
* is what every other write in this library is: a new generation. The current generation is streamed chunk by
|
|
7
7
|
* chunk through the ascending writer — every chunk decoded, range-checked and re-encoded, the one holding the id
|
|
8
|
-
* with that bit cleared — then published fenced on the generation it streamed, and that generation is collected
|
|
9
|
-
* (`keep: 0`), so the bit is **physically gone from the bucket when this
|
|
10
|
-
*
|
|
8
|
+
* with that bit cleared — then published fenced on the generation it streamed, and that generation is collected
|
|
9
|
+
* immediately (`keep: 0`), so when it answers `erased: true` the bit is **physically gone from the bucket when this
|
|
10
|
+
* returns** (a noncurrent version, a replica or a backup keeps the old object until its own lifecycle removes it).
|
|
11
|
+
* Bounded memory: a window of 32 chunk reads ahead of the writer, never the whole segment.
|
|
11
12
|
*
|
|
12
13
|
* The rewrite is the same generation without one id, so it keeps everything else: the new object carries the source's
|
|
13
14
|
* metadata as it is, and the row's summary of it, built from what was written, counts one id fewer and holds the same
|
|
@@ -21,12 +22,17 @@
|
|
|
21
22
|
* leaves generations above the pointer that were once current and can be made current again, so a holder can sit
|
|
22
23
|
* above the pointer as well as below it, and there can be several. Before it reports `erased: true` the call
|
|
23
24
|
* therefore lists the bucket once more and reads every generation it has not already seen to be clean; any that
|
|
24
|
-
* still holds the id makes it report or throw instead.
|
|
25
|
-
*
|
|
25
|
+
* still holds the id makes it report or throw instead. A number can be taken again once its object is deleted, so a
|
|
26
|
+
* generation it saw clean is read again too, but for two: the one it found current and its own rewrite, each skipped
|
|
27
|
+
* only while the row, read after that listing, names it with the footer checksum of the object the call read or wrote.
|
|
28
|
+
* In the ordinary case that list returns one generation, the current one, which the row names: one `list` and one
|
|
29
|
+
* read of the row.
|
|
26
30
|
*
|
|
27
31
|
* What it does NOT reach: backups, replicas and noncurrent object versions hold the old object until their own
|
|
28
|
-
* lifecycle removes it. For an at-rest guarantee that
|
|
29
|
-
* (`destroySegment`), which is per segment — a per-id shred is infeasible because one DEK covers the whole segment
|
|
32
|
+
* lifecycle removes it. For an at-rest guarantee that reaches those, use encryption plus crypto-shred
|
|
33
|
+
* (`destroySegment`), which is per segment — a per-id shred is infeasible because one DEK covers the whole segment —
|
|
34
|
+
* and is complete once no retained copy of the segment's registry row holds its wrapped key, or every KEK that
|
|
35
|
+
* wrapped it is destroyed.
|
|
30
36
|
*
|
|
31
37
|
* Concurrency. This is a **read-modify-write**, not a plain forward-only publish, and that distinction is where
|
|
32
38
|
* its correctness lives: the new generation is `from` minus one bit, so it is a valid successor to `from` and to
|
|
@@ -36,8 +42,8 @@
|
|
|
36
42
|
* incarnation of a name, and a name that is retired and re-created restarts at `0`, so the number alone matched
|
|
37
43
|
* a different segment entirely. Anything else — a load that published in the meantime, or another
|
|
38
44
|
* erasure that got there first — is reported as `reason: 'superseded'`, `erased: false`, and the caller re-runs
|
|
39
|
-
* against the new generation.
|
|
40
|
-
* `WriteConflictError
|
|
45
|
+
* against the new generation. So is a writer that claims the same generation *number* first (write-once): the put
|
|
46
|
+
* is refused with a `WriteConflictError`, which is read here as the lost race it is.
|
|
41
47
|
*
|
|
42
48
|
* The same answer covers the read half, and for the same reason. An erasure collects with `keep: 0`, which takes
|
|
43
49
|
* every generation below its new pointer — so a racing erasure can delete the generation this call is streaming,
|
|
@@ -176,8 +182,8 @@ export interface EraseIdResult {
|
|
|
176
182
|
*/
|
|
177
183
|
readonly erased: boolean;
|
|
178
184
|
/**
|
|
179
|
-
* Why the id was not erased, when `erased` is false. `'absent'` (no registry row), `'destroyed'` (a
|
|
180
|
-
*
|
|
185
|
+
* Why the id was not erased, when `erased` is false. `'absent'` (no registry row), `'destroyed'` (a tombstone; its
|
|
186
|
+
* objects are searched first), `'no-generation'` (a row with no generation yet when the call read it, its first
|
|
181
187
|
* read: its bucket can hold first loads' objects, none of them found to hold the id, and `collected` names any this
|
|
182
188
|
* call deleted unsearched, see below),
|
|
183
189
|
* `'not-member'` (no
|
|
@@ -232,7 +238,7 @@ export interface EraseIdResult {
|
|
|
232
238
|
* a missing holder as "someone else got there", never as a failure.
|
|
233
239
|
*
|
|
234
240
|
* A call that cannot establish the claim **throws** `WriteConflictError` or the underlying fault rather than
|
|
235
|
-
* report `erased: true` over bytes that are still there. Four ways that happens: a Storage `delete` fault; a
|
|
241
|
+
* report `erased: true` over bytes that are still there. Four ways that happens: a Storage `delete` fault the call cannot settle by reading the object back; a
|
|
236
242
|
* collection pass that could not prove the segment was still the same one (which an ordinary retirement landing
|
|
237
243
|
* mid-call is enough to cause); the same refusal on the path where the current generation did not hold the id
|
|
238
244
|
* and nothing was published at all; and a generation still holding the id when the bucket is listed at the end
|
|
@@ -242,6 +248,13 @@ export interface EraseIdResult {
|
|
|
242
248
|
* rewrite discards its own object, when it gets no answer that reading the row can settle, throws the registry's
|
|
243
249
|
* `TransientError`, and that object is not deleted.
|
|
244
250
|
*
|
|
251
|
+
* A delete that gets no answer is settled by reading its key: gone, it counts as deleted and is listed in
|
|
252
|
+
* `collected`. Another object under the number, for a delete conditioned on the version searched, is not counted as
|
|
253
|
+
* deleted: the receipt then rests on the listing of the bucket, and says `erased: true` only when that object does
|
|
254
|
+
* not hold the id. When that read fails too, or the delete lands after it, the call reports the fault, and a re-run
|
|
255
|
+
* finds the object gone and lists nothing for the segment: keep the error with the ledger as the record of the
|
|
256
|
+
* deletion.
|
|
257
|
+
*
|
|
245
258
|
* A rollback that lands between the call's read of the row and one of its deletes, onto the generation being deleted,
|
|
246
259
|
* leaves the pointer on an object that is gone. The call finds it at its next read of the row, or by reading the row
|
|
247
260
|
* once more after its last delete, and **throws `NotFoundError`** rather than report anything: every read of the
|
|
@@ -277,3 +290,13 @@ export interface EraseIdResult {
|
|
|
277
290
|
export declare function eraseIdFromSegment(ref: SegmentRef, id: number, deps: EraseIdDeps, options?: {
|
|
278
291
|
audit?: IAuditSink;
|
|
279
292
|
}): Promise<EraseIdResult>;
|
|
293
|
+
/** What the search for an id in an object needs of the store: the objects, the codec, and the read bounds. */
|
|
294
|
+
export type SearchDeps = Pick<EraseIdDeps, 'storage' | 'codec' | 'maxBitmapBytes' | 'readRetry'>;
|
|
295
|
+
/**
|
|
296
|
+
* The newest generation left under a tombstone that holds `id` in the clear, or `null`: the search
|
|
297
|
+
* {@link eraseIdFromSegment} makes under a tombstone, without its deletes. The caller has read the row as `destroyed`;
|
|
298
|
+
* this reads no row. It lists the bucket once and opens objects newest first, a few at a time, stopping at the first
|
|
299
|
+
* that holds the id; an object sealed under a key is no holder. It writes and deletes nothing. A fault on a read is
|
|
300
|
+
* thrown, after the read retry in `deps.readRetry` (the listing is retried too), never read as "not found".
|
|
301
|
+
*/
|
|
302
|
+
export declare function findIdUnderTombstone(ref: SegmentRef, id: number, deps: SearchDeps): Promise<number | null>;
|