@cloudbitmaps/core 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/dist/chunk-7GT632UF.js +641 -0
- package/dist/chunk-7GT632UF.js.map +7 -0
- package/dist/chunk-CGILDXGP.js +73 -0
- package/dist/chunk-CGILDXGP.js.map +7 -0
- package/dist/chunk-GKHERVVQ.js +121 -0
- package/dist/chunk-GKHERVVQ.js.map +7 -0
- package/dist/{chunk-OSU7W2Y3.js → chunk-MVUD4R72.js} +200 -756
- package/dist/chunk-MVUD4R72.js.map +7 -0
- package/dist/core/audit.d.ts +53 -15
- package/dist/core/bit-route.d.ts +2 -0
- package/dist/core/blob.d.ts +4 -0
- package/dist/core/budget.d.ts +8 -3
- package/dist/core/check.d.ts +49 -0
- package/dist/core/combine-many.d.ts +11 -0
- package/dist/core/consistency.d.ts +11 -2
- package/dist/core/cooperative.d.ts +7 -0
- package/dist/core/crbm-storage-source.d.ts +7 -4
- package/dist/core/crypto.d.ts +5 -0
- package/dist/core/discover.d.ts +5 -3
- package/dist/core/engine.d.ts +4 -4
- package/dist/core/erase-id.d.ts +35 -12
- package/dist/core/erasure.d.ts +61 -27
- package/dist/core/errors.d.ts +30 -0
- package/dist/core/leases.d.ts +2 -0
- package/dist/core/load-input.d.ts +48 -0
- package/dist/core/load.d.ts +23 -2
- package/dist/core/metrics.d.ts +21 -0
- package/dist/core/pinned-storage-source.d.ts +27 -8
- package/dist/core/pointer-id.d.ts +9 -0
- package/dist/core/ports.d.ts +100 -7
- package/dist/core/preflight.d.ts +80 -0
- package/dist/core/retention-sweep.d.ts +60 -15
- package/dist/core/retention.d.ts +17 -6
- package/dist/core/retry.d.ts +1 -2
- package/dist/core/rollback.d.ts +22 -12
- package/dist/core/validate.d.ts +33 -2
- package/dist/driver-kit.d.ts +3 -3
- package/dist/driver-kit.js +61 -20
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +24 -1
- package/dist/drivers/_shared/options.d.ts +15 -7
- package/dist/drivers/backends.d.ts +17 -2
- package/dist/drivers/crypto.d.ts +17 -2
- package/dist/export/index.d.ts +16 -2
- package/dist/index.d.ts +10 -9
- package/dist/index.js +1743 -1701
- package/dist/index.js.map +4 -4
- package/dist/node.d.ts +14 -0
- package/dist/node.js +1064 -0
- package/dist/node.js.map +7 -0
- package/package.json +11 -2
- package/dist/chunk-OSU7W2Y3.js.map +0 -7
package/dist/core/erasure.d.ts
CHANGED
|
@@ -1,33 +1,51 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Whole-segment erasure: crypto-shred (`destroySegment`, `eraseNamespace`) and disposal (`dropSegment`).
|
|
3
3
|
*
|
|
4
|
-
* GDPR "right to erasure" on immutable, backed-up storage: you can't delete a `.crbm` object from every
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* GDPR "right to erasure" on immutable, backed-up storage: you can't delete a `.crbm` object from every backup, but you
|
|
5
|
+
* can **delete its key**. `destroySegment` drops the segment's wrapped DEK(s) from its current registry row (a CAS to a
|
|
6
|
+
* `destroyed` tombstone). Every copy of the encrypted Storage bytes, backups and WORM copies included, is then
|
|
7
|
+
* unreadable once no retained copy of the row still holds the wrapped key (a noncurrent version, a backup or a
|
|
8
|
+
* point-in-time copy of the row), or once every KEK that wrapped it is destroyed; until then a registry restore to a
|
|
9
|
+
* point before the shred makes the segment readable again. **Immediate**, and it cannot be undone through the library
|
|
10
|
+
* (a registry restore to before it can), so it's gated behind an explicit confirmation (the
|
|
11
|
+
* segment's name; the namespace is not checked, so build the ref's `namespace` from a value you have checked). A `destroyed` segment reads as empty: no generation resolves through a destroyed row. Only an
|
|
12
|
+
* **encrypted** segment is crypto-shredded; a cleartext one has no key to shred, and `allowCleartext` tombstones it
|
|
13
|
+
* with its objects left readable.
|
|
10
14
|
*
|
|
11
|
-
* The tombstone is also the fence every writer respects: a load refuses
|
|
12
|
-
*
|
|
13
|
-
*
|
|
15
|
+
* The tombstone is also the fence every writer respects: a load refuses a `destroyed` row, so a load racing an erasure
|
|
16
|
+
* cannot resurrect the segment. A single id's erasure is a different operation — `eraseIdFromSegment` rewrites the
|
|
17
|
+
* generation without it.
|
|
14
18
|
*/
|
|
15
19
|
import { type IAuditSink } from './audit.js';
|
|
16
20
|
import type { ChurnDeps } from './leases.js';
|
|
17
21
|
import type { IStorageDriver, IRegistryDriver, RegistryRecord, SegmentRef } from './ports.js';
|
|
22
|
+
/**
|
|
23
|
+
* What the erasure calls need: the registry whose rows they tombstone, and, optionally, the clock and rng they wait out
|
|
24
|
+
* lease churn with.
|
|
25
|
+
*/
|
|
18
26
|
export interface EraseDeps extends ChurnDeps {
|
|
27
|
+
/** The registry holding the segments' rows. */
|
|
19
28
|
readonly registry: IRegistryDriver;
|
|
20
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* What destroying one segment did: a crypto-shred, a cleartext tombstone (the bytes stay readable), nothing for a
|
|
32
|
+
* segment already destroyed or absent, or, with `destroyed: false` and a `reason`, a segment left as it was, which may
|
|
33
|
+
* still hold data. The result of `destroySegment`, and one entry of what `eraseNamespace` returns.
|
|
34
|
+
*/
|
|
21
35
|
export interface DestroyResult {
|
|
36
|
+
/** The segment's name. */
|
|
22
37
|
readonly segment: string;
|
|
38
|
+
/** The segment's namespace, or `undefined` for the default one. */
|
|
23
39
|
readonly namespace?: string;
|
|
24
40
|
/** True iff the segment is now a `destroyed` tombstone (incl. the idempotent already-destroyed case). */
|
|
25
41
|
readonly destroyed: boolean;
|
|
26
42
|
/**
|
|
27
43
|
* True iff **this call** performed a genuine crypto-shred — an encrypted segment whose wrapped DEK(s) were
|
|
28
|
-
* dropped, so its Storage bytes
|
|
29
|
-
*
|
|
30
|
-
*
|
|
44
|
+
* dropped from its current registry row, so its Storage bytes, in any copy, are unreadable once no retained copy
|
|
45
|
+
* of the row holds the key, or every KEK that wrapped it is destroyed. False for a cleartext tombstone (bytes
|
|
46
|
+
* remain readable), for the idempotent already-destroyed case, and for an absent segment. This is what the
|
|
47
|
+
* `segment.erase` audit event keys off — a cleartext "erase" destroys no key. A shred that throws `TransientError`
|
|
48
|
+
* after its read-back also failed may have landed; its re-run answers `'already'` with `cryptoShredded: false`.
|
|
31
49
|
*/
|
|
32
50
|
readonly cryptoShredded: boolean;
|
|
33
51
|
/**
|
|
@@ -38,14 +56,20 @@ export interface DestroyResult {
|
|
|
38
56
|
*
|
|
39
57
|
* The last two come with `destroyed: false` and mean **the segment still holds data**. A namespace erase
|
|
40
58
|
* returns them rather than throwing, so its entries have to be inspected — see {@link eraseNamespace}.
|
|
59
|
+
*
|
|
60
|
+
* A shred that throws `TransientError` after its read-back also failed may have landed; its re-run answers
|
|
61
|
+
* `'already'` with `cryptoShredded: false`.
|
|
41
62
|
*/
|
|
42
63
|
readonly reason?: string;
|
|
43
64
|
}
|
|
44
65
|
/**
|
|
45
|
-
* Crypto-shred one segment.
|
|
46
|
-
*
|
|
47
|
-
* via `allowCleartext` (there's no key to shred; its
|
|
48
|
-
* segment resolving).
|
|
66
|
+
* Crypto-shred one segment. It cannot be undone through the library (a registry restore to a point before it can).
|
|
67
|
+
* `confirmSegment` must equal `ref.segment` (a guard against an accidental destroy of another name; it checks the
|
|
68
|
+
* name only, not the namespace, so build the ref's `namespace` from a value you have checked). A non-encrypted segment is rejected unless you opt in via `allowCleartext` (there's no key to shred; its
|
|
69
|
+
* Storage bytes stay readable — the tombstone only stops the segment resolving).
|
|
70
|
+
*
|
|
71
|
+
* The options are `confirmSegment`, `allowCleartext` and `audit`. Any other key (`dryRun`, which this call does not
|
|
72
|
+
* take, included) and an `allowCleartext` that is not a boolean throw `ValidationError` before the registry is read.
|
|
49
73
|
*/
|
|
50
74
|
export declare function destroySegment(ref: SegmentRef, deps: EraseDeps, options: {
|
|
51
75
|
confirmSegment: string;
|
|
@@ -53,14 +77,16 @@ export declare function destroySegment(ref: SegmentRef, deps: EraseDeps, options
|
|
|
53
77
|
audit?: IAuditSink;
|
|
54
78
|
}): Promise<DestroyResult>;
|
|
55
79
|
/**
|
|
56
|
-
* Crypto-shred every segment in a namespace.
|
|
57
|
-
* Returns a per-segment result (skips cleartext segments
|
|
80
|
+
* Crypto-shred every segment in a namespace. It cannot be undone through the library (a registry restore to a point
|
|
81
|
+
* before it can). `confirmNamespace` must equal `namespace`. Returns a per-segment result (skips cleartext segments
|
|
82
|
+
* unless `allowCleartext`).
|
|
58
83
|
*
|
|
59
84
|
* **Bounded.** It lists the whole namespace before it destroys anything, and holds the listing resident, so the
|
|
60
85
|
* listing is capped at `maxScanSegments` (default 250,000), the ceiling every
|
|
61
86
|
* other fleet scan keeps. A namespace over it throws `BudgetExceededError` **before any segment is destroyed**,
|
|
62
87
|
* never part-way through; raise `maxScanSegments` when the namespace really is that large and the memory is there.
|
|
63
88
|
* A `maxScanSegments` that is not a safe whole number of at least 1 throws `ValidationError`, again before anything is destroyed.
|
|
89
|
+
* So does any option it does not take (`dryRun` included) and an `allowCleartext` that is not a boolean.
|
|
64
90
|
*/
|
|
65
91
|
export declare function eraseNamespace(namespace: string, deps: EraseDeps, options: {
|
|
66
92
|
confirmNamespace: string;
|
|
@@ -76,8 +102,11 @@ export declare function eraseNamespace(namespace: string, deps: EraseDeps, optio
|
|
|
76
102
|
export interface DropDeps extends EraseDeps {
|
|
77
103
|
readonly storage: IStorageDriver;
|
|
78
104
|
}
|
|
105
|
+
/** What `dropSegment` did to one segment, or, on a dry run, would do. */
|
|
79
106
|
export interface DropResult {
|
|
107
|
+
/** The segment's name. */
|
|
80
108
|
readonly segment: string;
|
|
109
|
+
/** The segment's namespace, or `undefined` for the default one. */
|
|
81
110
|
readonly namespace?: string;
|
|
82
111
|
/**
|
|
83
112
|
* **True iff this segment is now empty as a result of this call — or already was.** This is the field a
|
|
@@ -111,14 +140,16 @@ export interface DropResult {
|
|
|
111
140
|
readonly wouldDelete?: readonly number[];
|
|
112
141
|
/**
|
|
113
142
|
* On a dry run, whether the real drop would **also** crypto-shred (i.e. the segment is encrypted).
|
|
114
|
-
* `undefined` on a real run. This is the one fact in the preview that
|
|
115
|
-
*
|
|
143
|
+
* `undefined` on a real run. This is the one fact in the preview that reaches every copy of the objects, backups
|
|
144
|
+
* and WORM copies included (once no retained copy of the row holds the key, or every KEK that wrapped it is
|
|
145
|
+
* destroyed) — a dry run that did not surface it would preview the recoverable half only.
|
|
116
146
|
*/
|
|
117
147
|
readonly wouldCryptoShred?: boolean;
|
|
118
148
|
/**
|
|
119
|
-
* True iff the segment was encrypted and its wrapped DEK(s) were dropped
|
|
120
|
-
* are unreadable
|
|
121
|
-
*
|
|
149
|
+
* True iff the segment was encrypted and its wrapped DEK(s) were dropped from its current registry row as part of
|
|
150
|
+
* this call — so its bytes are unreadable in every copy, backups included, once no retained copy of the row holds
|
|
151
|
+
* the key or every KEK that wrapped it is destroyed, not merely deleted from the bucket. Deleting an object does
|
|
152
|
+
* not reach a noncurrent version, a replica, or a PITR snapshot; removing the key does, on those terms.
|
|
122
153
|
*/
|
|
123
154
|
readonly cryptoShredded: boolean;
|
|
124
155
|
/**
|
|
@@ -133,14 +164,17 @@ export interface DropResult {
|
|
|
133
164
|
readonly reason?: string;
|
|
134
165
|
}
|
|
135
166
|
/**
|
|
136
|
-
* **Dispose of a segment: tombstone it, then delete its Storage objects.**
|
|
167
|
+
* **Dispose of a segment: tombstone it, then delete its Storage objects.** It cannot be undone through the library
|
|
168
|
+
* (a restore from the registry's and the bucket's versions or backups can).
|
|
137
169
|
*
|
|
138
170
|
* WHY THIS EXISTS, given {@link destroySegment} already erases. Because `destroySegment` answers a *compliance*
|
|
139
171
|
* question and this one answers an *operational* question, and they are not the same:
|
|
140
172
|
*
|
|
141
|
-
* - `destroySegment` **crypto-shreds** — it
|
|
142
|
-
*
|
|
143
|
-
*
|
|
173
|
+
* - `destroySegment` **crypto-shreds** — it removes the key from the segment's current registry row, so the bytes
|
|
174
|
+
* are unreadable everywhere, immutable backups included, once no retained copy of the row holds the key or every
|
|
175
|
+
* KEK that wrapped it is destroyed: the only erasure that reaches WORM copies of the objects. But it **leaves the
|
|
176
|
+
* objects in the bucket**, still billed, and it *requires* encryption because a cleartext segment has no key to
|
|
177
|
+
* discard.
|
|
144
178
|
* - `dropSegment` **removes the storage**. It works on a cleartext segment, and on an encrypted one it *also*
|
|
145
179
|
* drops the DEKs, so it is a strict superset there.
|
|
146
180
|
*
|
package/dist/core/errors.d.ts
CHANGED
|
@@ -24,6 +24,7 @@ declare const TRANSIENT_BRAND: unique symbol;
|
|
|
24
24
|
export declare class CloudRoaringError extends Error {
|
|
25
25
|
/** Cross-bundle brand — see {@link errorKind}. Non-enumerable-ish (symbol key ⇒ not in JSON). */
|
|
26
26
|
readonly [ERROR_BRAND]: true;
|
|
27
|
+
/** Builds the error from `message`, and sets `name` to the class's name as written. */
|
|
27
28
|
constructor(message: string);
|
|
28
29
|
}
|
|
29
30
|
/**
|
|
@@ -59,6 +60,7 @@ export declare class LeaseExpiredError extends CloudRoaringError {
|
|
|
59
60
|
readonly until: number;
|
|
60
61
|
/** `'expired'` when the clock reached `until`, `'released'` after `release()`. */
|
|
61
62
|
readonly reason: 'expired' | 'released';
|
|
63
|
+
/** Builds the error from its `message`, the epoch-ms the lease ran until, and why it ended. */
|
|
62
64
|
constructor(message: string, until: number, reason: 'expired' | 'released');
|
|
63
65
|
}
|
|
64
66
|
/**
|
|
@@ -75,11 +77,13 @@ export declare class LeaseLimitError extends CloudRoaringError {
|
|
|
75
77
|
* `'stale-operand'`. Deterministic: run the call again against the current operand.
|
|
76
78
|
*/
|
|
77
79
|
export declare class StaleOperandError extends CloudRoaringError {
|
|
80
|
+
/** The stable code `'stale-operand'`, to match on. */
|
|
78
81
|
readonly code: "stale-operand";
|
|
79
82
|
/** The operand's name in the call that found it stale. */
|
|
80
83
|
readonly operand: string;
|
|
81
84
|
/** Why it is stale. */
|
|
82
85
|
readonly reason: 'moved' | 'erased';
|
|
86
|
+
/** Builds the error from its `message`, the operand's name and why it is stale. */
|
|
83
87
|
constructor(message: string, operand: string, reason: 'moved' | 'erased');
|
|
84
88
|
}
|
|
85
89
|
/**
|
|
@@ -134,6 +138,7 @@ export declare class BudgetExceededError extends CloudRoaringError {
|
|
|
134
138
|
* lower bound on the resident bytes it needs (`'maxBufferedBytes'`), or `limit + 1` (`'maxScanSegments'`).
|
|
135
139
|
*/
|
|
136
140
|
readonly spent: number;
|
|
141
|
+
/** Builds the error from its `message` and the `option`, `limit` and `spent` it reports. */
|
|
137
142
|
constructor(message: string, details: {
|
|
138
143
|
readonly option: 'budget.maxRequests' | 'maxBufferedBytes' | 'maxScanSegments';
|
|
139
144
|
readonly limit: number;
|
|
@@ -166,6 +171,7 @@ export declare class KeyUnavailableError extends CloudRoaringError {
|
|
|
166
171
|
export declare class TransientError extends CloudRoaringError {
|
|
167
172
|
/** A second brand so a transient fault is classifiable cross-bundle. */
|
|
168
173
|
readonly [TRANSIENT_BRAND]: true;
|
|
174
|
+
/** Builds the error from `message`; `options.cause`, when given, is kept as `cause`. */
|
|
169
175
|
constructor(message: string, options?: {
|
|
170
176
|
cause?: unknown;
|
|
171
177
|
});
|
|
@@ -174,6 +180,30 @@ export declare class TransientError extends CloudRoaringError {
|
|
|
174
180
|
export declare function isCloudRoaringError(err: unknown): err is CloudRoaringError;
|
|
175
181
|
/** An OCC conditional write/delete lost the race — retry the read-modify-write, don't fail. */
|
|
176
182
|
export declare function isWriteConflictError(err: unknown): err is WriteConflictError;
|
|
183
|
+
/**
|
|
184
|
+
* A {@link WriteConflictError} for a registry `create` or `compareAndSwap` that may have landed before it met the
|
|
185
|
+
* conflict, so the conflict does not prove the write was refused. Throw it for every conflict of a write your backend's
|
|
186
|
+
* client may have sent more than once and that you cannot prove did not land: the read-back finds another writer's
|
|
187
|
+
* write over the row (a row is overwritten, so that writer may have swapped in over this write after it landed), or
|
|
188
|
+
* finds the row gone, or the compare-and-swap itself is answered that the row is gone. Throw a plain
|
|
189
|
+
* `WriteConflictError` for a conflict that does prove it: a write sent once, or a check your driver made before sending
|
|
190
|
+
* anything. A registry that wraps another rethrows the conflict it was given, not a new one, which would lose the mark.
|
|
191
|
+
*
|
|
192
|
+
* It is a `WriteConflictError` in every other way: `errorKind` names it `'WriteConflictError'`, `instanceof` holds on
|
|
193
|
+
* the same copy of the package, and every caller that handles a conflict handles it. A load's publish reads the row
|
|
194
|
+
* again, as after any conflict, and takes the write as its own when the row shows it there. Otherwise the write is
|
|
195
|
+
* reported to the load, as a write with no answer is, so a load refused after it carries `unanswered: true` on its
|
|
196
|
+
* `segment.load-refused` audit event. The mark is a registered `Symbol.for` brand, read
|
|
197
|
+
* across copies of the package. It is lost when the error is cloned or copied into a new one, and a conflict raised
|
|
198
|
+
* anywhere but a registry write is read as the plain conflict it also is.
|
|
199
|
+
*/
|
|
200
|
+
export declare function unsettledConflict(message: string): WriteConflictError;
|
|
201
|
+
/**
|
|
202
|
+
* Whether `err` is a conflict a driver raised with {@link unsettledConflict}: a {@link WriteConflictError}, by brand
|
|
203
|
+
* and name as {@link isWriteConflictError} reads it, that carries the unsettled mark. It reads the registered brand,
|
|
204
|
+
* so it holds for an error from another copy of the package. Like the other predicates, no entry exports it.
|
|
205
|
+
*/
|
|
206
|
+
export declare function isUnsettledConflict(err: unknown): boolean;
|
|
177
207
|
/** A retryable transient infrastructure fault. The retry layer keys on this. */
|
|
178
208
|
export declare function isTransientError(err: unknown): err is TransientError;
|
|
179
209
|
/** A read came after its pin's lease ended. */
|
package/dist/core/leases.d.ts
CHANGED
|
@@ -40,9 +40,11 @@ export declare function liveLeases(record: RegistryRecord | null, now: number):
|
|
|
40
40
|
* Both are optional; without a sleep a retry follows at once.
|
|
41
41
|
*/
|
|
42
42
|
export interface ChurnDeps {
|
|
43
|
+
/** The clock whose `sleep` waits out lease churn. Without a `sleep`, a retry follows at once. */
|
|
43
44
|
readonly clock?: {
|
|
44
45
|
readonly sleep?: (ms: number) => Promise<void>;
|
|
45
46
|
} | undefined;
|
|
47
|
+
/** The source that spreads each wait over a share of its bound. Without one, each wait is its whole bound. */
|
|
46
48
|
readonly rng?: Rng | undefined;
|
|
47
49
|
}
|
|
48
50
|
/**
|
|
@@ -17,7 +17,12 @@ import type { CombinedChunk } from './engine.js';
|
|
|
17
17
|
* (`getSerializationSizeInBytes`) is asked first, so one over the cap is refused before it is serialized.
|
|
18
18
|
*/
|
|
19
19
|
export interface PortableBitmap {
|
|
20
|
+
/** The bitmap in the portable Roaring format. */
|
|
20
21
|
serialize(format: 'portable'): Uint8Array;
|
|
22
|
+
/**
|
|
23
|
+
* The size in bytes of `serialize('portable')`, when the bitmap can say without serializing. A load asks it first, so
|
|
24
|
+
* an input over the size cap is refused before it is serialized.
|
|
25
|
+
*/
|
|
21
26
|
getSerializationSizeInBytes?(format: 'portable'): number;
|
|
22
27
|
}
|
|
23
28
|
/**
|
|
@@ -53,6 +58,49 @@ export declare class ChunkLoadInput {
|
|
|
53
58
|
readonly chunks: AsyncIterable<CombinedChunk>;
|
|
54
59
|
constructor(chunks: AsyncIterable<CombinedChunk>);
|
|
55
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Mark `err` as raised by a load's input (its ids, a bitmap's encoding, a combine's chunks) rather than by the write,
|
|
63
|
+
* and return it. A write-once conflict is a lost race only when the write raised it: a source that reads another
|
|
64
|
+
* segment (through a pin, say) can throw this library's own `WriteConflictError`, which is the source's to report.
|
|
65
|
+
*/
|
|
66
|
+
export declare function raisedBySource<T>(err: T): T;
|
|
67
|
+
/** Whether a load's input raised `err` ({@link raisedBySource}). */
|
|
68
|
+
export declare function isSourceError(err: unknown): boolean;
|
|
69
|
+
/**
|
|
70
|
+
* Ids the caller says ascend (`sorted: true`), written chunk by chunk while they are read, during the object's upload,
|
|
71
|
+
* so a load holds one chunk and the writer's index rather than the whole generation. Made only by
|
|
72
|
+
* {@link sortedLoadInput}, from ids.
|
|
73
|
+
*
|
|
74
|
+
* The first error the ids raise, or their check raises (an id out of range, or lower than the one before it), or the
|
|
75
|
+
* chunker raises building a chunk, is kept here, and the load throws that same error. The driver's upload never sees
|
|
76
|
+
* it: the chunker throws a sentinel of its own into the writer instead, so no driver maps, redacts or probes the
|
|
77
|
+
* caller's error on its way out.
|
|
78
|
+
*/
|
|
79
|
+
export declare class SortedLoadInput {
|
|
80
|
+
readonly ids: Iterable<number> | AsyncIterable<number>;
|
|
81
|
+
private failure;
|
|
82
|
+
constructor(ids: Iterable<number> | AsyncIterable<number>);
|
|
83
|
+
/** Keep `error` as what the ids raised, unless an earlier one is kept. */
|
|
84
|
+
fail(error: unknown): void;
|
|
85
|
+
/** What the load throws for a write that failed with `err`: the error the ids raised, when they raised one. */
|
|
86
|
+
thrown(err: unknown): unknown;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Whether a load was asked for `sorted: true`. `undefined` and `false` mean no; anything else that is not a boolean is
|
|
90
|
+
* refused before any request, since `sorted: 'yes'` read as absent would hold the whole generation in memory instead.
|
|
91
|
+
*
|
|
92
|
+
* @throws {ValidationError} for a value that is not a boolean.
|
|
93
|
+
*/
|
|
94
|
+
export declare function wantsSorted(sorted: unknown): boolean;
|
|
95
|
+
/**
|
|
96
|
+
* The input a load reads under its `sorted` option: ids as {@link SortedLoadInput} when it is `true`, anything else
|
|
97
|
+
* unchanged.
|
|
98
|
+
*
|
|
99
|
+
* @throws {ValidationError} for a `sorted` that is not a boolean, and for `sorted: true` with `{ serialized }`,
|
|
100
|
+
* `{ bitmap }` or a bitmap of the flavor's own: those are written from their own chunks already, so the option would
|
|
101
|
+
* do nothing, and an option that does nothing is refused rather than ignored.
|
|
102
|
+
*/
|
|
103
|
+
export declare function sortedLoadInput(prepared: ReturnType<typeof prepareLoadInput>, sorted: unknown): ReturnType<typeof prepareLoadInput> | SortedLoadInput;
|
|
56
104
|
/**
|
|
57
105
|
* Check a load's input and decode a bitmap input, before the load makes any request.
|
|
58
106
|
*
|
package/dist/core/load.d.ts
CHANGED
|
@@ -140,6 +140,26 @@ export interface LoadOptions {
|
|
|
140
140
|
readonly metadata?: GenerationMetadata;
|
|
141
141
|
/** Audit sink for this load: it receives `segment.publish`, or `segment.load-refused` when the load is refused. */
|
|
142
142
|
readonly audit?: IAuditSink;
|
|
143
|
+
/**
|
|
144
|
+
* The ids ascend: write them as they are read, one chunk at a time, while the object uploads, so the load holds one
|
|
145
|
+
* chunk and the writer's index of the chunks written (up to about 7 MiB, about 8 MiB for an encrypted segment,
|
|
146
|
+
* derived, at the 65,536 chunks a segment can have) rather than the whole generation, whatever the segment's size.
|
|
147
|
+
* Repeats are fine (an id equal to the one before is skipped); an id lower than the one before throws
|
|
148
|
+
* `ValidationError`, naming its position and never the id. The object is the same, byte for byte, as an unsorted load
|
|
149
|
+
* of the same ids writes under the same write identity.
|
|
150
|
+
*
|
|
151
|
+
* For ids only (an iterable or an async iterable): with `{ serialized }`, `{ bitmap }` or a `RoaringBitmap32` of the
|
|
152
|
+
* flavor's own `roaring` (one from another copy loads as its ids, and takes `sorted`), and on core's
|
|
153
|
+
* `loadSegmentChunks`, it throws `ValidationError`, as does a value that is not a boolean, before any request.
|
|
154
|
+
*
|
|
155
|
+
* The ids are read after the segment's row is read and judged, and after its number and key are settled, so a load
|
|
156
|
+
* that refuses a destroyed or keyless segment reads none of them. They are read while the object uploads: an error
|
|
157
|
+
* they raise, or a refused id, fails the upload, and no object is committed, but the parts uploaded before it may
|
|
158
|
+
* already have been billed (see the production guide). That error reaches the caller as the very object raised,
|
|
159
|
+
* and the storage driver never sees it. The ids can be read once: a storage driver that runs its write again gets
|
|
160
|
+
* `IntegrityError`, and nothing is written.
|
|
161
|
+
*/
|
|
162
|
+
readonly sorted?: boolean;
|
|
143
163
|
}
|
|
144
164
|
/** The bounds a guard can refuse a generation for: every {@link LoadRefusal} but a lost race. */
|
|
145
165
|
export type GuardRefusal = Exclude<LoadRefusal, 'superseded'>;
|
|
@@ -170,8 +190,9 @@ export type LoadRefusal = 'empty' | 'min-cardinality' | 'min-retained' | 'max-gr
|
|
|
170
190
|
* writing: another load published, a retention change, a rollback or an erasure wrote the row, the row was
|
|
171
191
|
* deleted, or a row appeared where the load found none. Once another write has changed the row, the object stays in
|
|
172
192
|
* the bucket, still billed, until a later load's collection deletes it or the segment is dropped, because its
|
|
173
|
-
* generation number may by then name another writer's object; until it goes, a rollback onto it needs
|
|
174
|
-
* `
|
|
193
|
+
* generation number may by then name another writer's object; until it goes, a rollback onto it needs
|
|
194
|
+
* `allowUnknown`, as onto any generation the row does not know. Once the row is `destroyed` (crypto-shredded,
|
|
195
|
+
* destroyed in the clear or dropped), the object is deleted; once the row is gone, so is an object whose footer
|
|
175
196
|
* matches what this load wrote (another load's object of the same ids is not a match: the footer carries each write's
|
|
176
197
|
* own identity), and so is a cleartext one under a row with key material.
|
|
177
198
|
*/
|
package/dist/core/metrics.d.ts
CHANGED
|
@@ -136,6 +136,10 @@ export type MetricEvent = {
|
|
|
136
136
|
* caller — `CloudRoaring` wraps a user sink so that a buggy sink can't break I/O.
|
|
137
137
|
*/
|
|
138
138
|
export interface IMetricsSink {
|
|
139
|
+
/**
|
|
140
|
+
* Receives one event, synchronously. What it throws, or an async one's rejection, is swallowed and never breaks the
|
|
141
|
+
* call it observes.
|
|
142
|
+
*/
|
|
139
143
|
onEvent(event: MetricEvent): void;
|
|
140
144
|
}
|
|
141
145
|
/** The default sink: discards everything, zero overhead. Used whenever no sink is wired. */
|
|
@@ -147,23 +151,37 @@ export declare const NOOP_METRICS: IMetricsSink;
|
|
|
147
151
|
export declare function safeMetrics(sink: IMetricsSink): IMetricsSink;
|
|
148
152
|
/** Accumulated totals — the shape returned by {@link CountingMetricsSink.snapshot}. */
|
|
149
153
|
export interface MetricsSnapshot {
|
|
154
|
+
/**
|
|
155
|
+
* Chunk requests to storage: how many, the bytes they returned, and the time they took in ms, transient-retry backoff
|
|
156
|
+
* included.
|
|
157
|
+
*/
|
|
150
158
|
readonly storage: {
|
|
151
159
|
readonly gets: number;
|
|
152
160
|
readonly bytes: number;
|
|
153
161
|
readonly totalMs: number;
|
|
154
162
|
};
|
|
163
|
+
/**
|
|
164
|
+
* Decoded-chunk cache hits and misses. A chunk counted without a lookup is a miss; `materializeMany` reads no cache, so
|
|
165
|
+
* it counts none.
|
|
166
|
+
*/
|
|
155
167
|
readonly cache: {
|
|
156
168
|
readonly hits: number;
|
|
157
169
|
readonly misses: number;
|
|
158
170
|
};
|
|
171
|
+
/** Retries taken after a transient fault. */
|
|
159
172
|
readonly retries: {
|
|
160
173
|
readonly transient: number;
|
|
161
174
|
};
|
|
175
|
+
/**
|
|
176
|
+
* Chunk-aligned combines of every operator (intersect, union and andNot): how many ran, the chunk keys they fetched
|
|
177
|
+
* and the keys they skipped without a fetch.
|
|
178
|
+
*/
|
|
162
179
|
readonly intersect: {
|
|
163
180
|
readonly calls: number;
|
|
164
181
|
readonly fetchedChunks: number;
|
|
165
182
|
readonly skippedChunks: number;
|
|
166
183
|
};
|
|
184
|
+
/** By operation name, how many calls were timed and their total time in ms. */
|
|
167
185
|
readonly ops: Readonly<Record<MetricOpName, {
|
|
168
186
|
readonly count: number;
|
|
169
187
|
readonly totalMs: number;
|
|
@@ -181,7 +199,10 @@ export declare class CountingMetricsSink implements IMetricsSink {
|
|
|
181
199
|
private intersect;
|
|
182
200
|
private ops;
|
|
183
201
|
private static zeroOps;
|
|
202
|
+
/** Adds the event to the totals. An `advisory` event is a note, not a quantity, and adds nothing. */
|
|
184
203
|
onEvent(event: MetricEvent): void;
|
|
204
|
+
/** A copy of the totals so far, which later events do not change. */
|
|
185
205
|
snapshot(): MetricsSnapshot;
|
|
206
|
+
/** Sets every total back to zero. */
|
|
186
207
|
reset(): void;
|
|
187
208
|
}
|
|
@@ -27,15 +27,34 @@ import type { CrbmStorageChunkSource } from './crbm-storage-source.js';
|
|
|
27
27
|
* What a pin holds for one segment: the generation, the version identifying those exact bytes, and the pinned
|
|
28
28
|
* object's fingerprint, so a read can tell that the generation it opens is still that object.
|
|
29
29
|
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* It is one of two shapes, told apart by `generation`: a pin of a generation holds all three fields, and a pin of a
|
|
31
|
+
* segment that had no generation holds `null` in all three. Narrowing on `generation` (or on either other field)
|
|
32
|
+
* narrows the record whole, so a numbered pin passes to `pinAt` as it is:
|
|
33
|
+
*
|
|
34
|
+
* ```ts
|
|
35
|
+
* const at = first.pinnedAt;
|
|
36
|
+
* if (at !== undefined && at.generation !== null) await store.segment('active-30d').pinAt(at);
|
|
37
|
+
* ```
|
|
38
|
+
*
|
|
39
|
+
* `fingerprint` is never `null` for a pin of a generation: without it a read could not tell a name purged and loaded
|
|
40
|
+
* again from the segment it pinned. `seg.pin()` always records it, and a pin of a generation without a string
|
|
41
|
+
* `version` and `fingerprint` is refused with a `ValidationError`.
|
|
33
42
|
*/
|
|
34
|
-
export
|
|
35
|
-
|
|
36
|
-
readonly
|
|
37
|
-
|
|
38
|
-
|
|
43
|
+
export type PinnedAt = {
|
|
44
|
+
/** The pinned generation. */
|
|
45
|
+
readonly generation: number;
|
|
46
|
+
/** The version of that generation's object. */
|
|
47
|
+
readonly version: string;
|
|
48
|
+
/** The fingerprint of that generation's object. */
|
|
49
|
+
readonly fingerprint: string;
|
|
50
|
+
} | {
|
|
51
|
+
/** `null`: the segment had no generation when it was pinned. */
|
|
52
|
+
readonly generation: null;
|
|
53
|
+
/** `null`, as `generation` is. */
|
|
54
|
+
readonly version: null;
|
|
55
|
+
/** `null`, as `generation` is. */
|
|
56
|
+
readonly fingerprint: null;
|
|
57
|
+
};
|
|
39
58
|
export declare class PinnedStorageChunkSource implements StorageChunkSource {
|
|
40
59
|
private readonly inner;
|
|
41
60
|
/**
|
|
@@ -17,3 +17,12 @@ export declare function renewPointer(row: Pick<RegistryRecord, 'currentGen'>): R
|
|
|
17
17
|
* cannot key a cache on it. A TypeScript registry fails to compile instead; this catches one in JavaScript.
|
|
18
18
|
*/
|
|
19
19
|
export declare function pointerIdOf(row: RegistryRecord): Token;
|
|
20
|
+
/**
|
|
21
|
+
* Whether `now` shows a write this call sent as the one that put the pointer where it is: an active row of the same
|
|
22
|
+
* incarnation as `before` (a `create` has none to compare), whose pointer names `generation` and whose `pointerId` is
|
|
23
|
+
* the token of one of the writes in `sent`, which a registry that reports its writes' tokens gave before sending each.
|
|
24
|
+
* The token decides and nothing else is asked, so another writer's write of the same pointer is not taken for this
|
|
25
|
+
* call's. A write that renews the `pointerId` over this call's own landed write (an erasure's renewal, or a write of
|
|
26
|
+
* the pointer back onto the same generation) reads as not this call's: the two cannot be told apart. Pure: no I/O.
|
|
27
|
+
*/
|
|
28
|
+
export declare function pointerSetBy(now: RegistryRecord | null, before: RegistryRecord | null, generation: number, sent: ReadonlySet<Token>): boolean;
|