@cloudbitmaps/core 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -8
- package/dist/chunk-7GT632UF.js +641 -0
- package/dist/chunk-7GT632UF.js.map +7 -0
- package/dist/chunk-CGILDXGP.js +73 -0
- package/dist/chunk-CGILDXGP.js.map +7 -0
- package/dist/chunk-GKHERVVQ.js +121 -0
- package/dist/chunk-GKHERVVQ.js.map +7 -0
- package/dist/{chunk-IGLIZBR7.js → chunk-MVUD4R72.js} +253 -765
- package/dist/chunk-MVUD4R72.js.map +7 -0
- package/dist/core/audit.d.ts +53 -15
- package/dist/core/bit-route.d.ts +9 -0
- package/dist/core/blob.d.ts +4 -0
- package/dist/core/budget.d.ts +8 -3
- package/dist/core/check.d.ts +27 -0
- package/dist/core/codec.d.ts +1 -1
- package/dist/core/combine-expr.d.ts +9 -5
- package/dist/core/combine-feed.d.ts +11 -3
- package/dist/core/combine-many.d.ts +28 -13
- package/dist/core/consistency.d.ts +11 -2
- package/dist/core/cooperative.d.ts +7 -0
- package/dist/core/crbm-storage-source.d.ts +7 -4
- package/dist/core/crypto.d.ts +5 -0
- package/dist/core/determinism.d.ts +2 -1
- package/dist/core/discover.d.ts +5 -3
- package/dist/core/engine.d.ts +4 -4
- package/dist/core/erase-id.d.ts +52 -19
- package/dist/core/erasure.d.ts +62 -28
- package/dist/core/errors.d.ts +83 -9
- package/dist/core/leases.d.ts +2 -0
- package/dist/core/load-input.d.ts +48 -0
- package/dist/core/load.d.ts +71 -16
- package/dist/core/metrics.d.ts +22 -1
- package/dist/core/pinned-storage-source.d.ts +27 -8
- package/dist/core/pointer-id.d.ts +9 -0
- package/dist/core/ports.d.ts +107 -13
- package/dist/core/preflight.d.ts +80 -0
- package/dist/core/registry-scan.d.ts +0 -5
- package/dist/core/retention-sweep.d.ts +61 -16
- package/dist/core/retention.d.ts +18 -7
- package/dist/core/retry.d.ts +5 -2
- package/dist/core/rollback.d.ts +23 -12
- package/dist/core/validate.d.ts +33 -2
- package/dist/driver-kit.d.ts +9 -4
- package/dist/driver-kit.js +63 -27
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +24 -1
- package/dist/drivers/_shared/options.d.ts +15 -7
- package/dist/drivers/backends.d.ts +19 -2
- package/dist/drivers/crypto.d.ts +18 -3
- package/dist/export/index.d.ts +25 -10
- package/dist/index.d.ts +19 -17
- package/dist/index.js +2457 -2283
- package/dist/index.js.map +4 -4
- package/dist/node.d.ts +14 -0
- package/dist/node.js +1064 -0
- package/dist/node.js.map +7 -0
- package/package.json +16 -2
- package/dist/chunk-IGLIZBR7.js.map +0 -7
- package/dist/core/absent-operands.d.ts +0 -8
- package/dist/core/bulk-load.d.ts +0 -148
- package/dist/core/cas-retry.d.ts +0 -27
- package/dist/core/chunk-checks.d.ts +0 -39
- package/dist/core/chunk-stream.d.ts +0 -22
- package/dist/core/chunk-window.d.ts +0 -37
- package/dist/core/combine-ledger.d.ts +0 -43
- package/dist/core/crbm/crc32c.d.ts +0 -10
- package/dist/core/crbm/fingerprint.d.ts +0 -23
- package/dist/core/crbm/format.d.ts +0 -150
- package/dist/core/crbm/frame.d.ts +0 -10
- package/dist/core/crbm/plan-reads.d.ts +0 -50
- package/dist/core/crbm/varint.d.ts +0 -14
- package/dist/core/crbm/writer.d.ts +0 -72
- package/dist/core/crbm-open.d.ts +0 -154
- package/dist/core/crbm-write.d.ts +0 -75
- package/dist/core/due-index.d.ts +0 -53
- package/dist/core/generation-gc.d.ts +0 -293
- package/dist/core/generation-list.d.ts +0 -11
- package/dist/core/id-window.d.ts +0 -46
- package/dist/core/invalidation-scope.d.ts +0 -77
- package/dist/core/item-pull.d.ts +0 -29
- package/dist/core/known-generations-shape.d.ts +0 -13
- package/dist/core/known-generations.d.ts +0 -147
- package/dist/core/ledger-seed.d.ts +0 -17
- package/dist/core/metadata.d.ts +0 -42
- package/dist/core/order.d.ts +0 -2
- package/dist/core/publish.d.ts +0 -207
- package/dist/core/reader-defaults.d.ts +0 -29
- package/dist/core/summary.d.ts +0 -110
- package/dist/core/token.d.ts +0 -41
- package/dist/drivers/_shared/entropy.d.ts +0 -5
- package/dist/drivers/_shared/registry.d.ts +0 -107
- package/dist/drivers/localfs/exact-case.d.ts +0 -34
- package/dist/drivers/localfs/fs-util.d.ts +0 -55
- package/dist/drivers/localfs/paths.d.ts +0 -40
- package/dist/drivers/localfs/registry.d.ts +0 -50
- package/dist/drivers/localfs/storage.d.ts +0 -28
- package/dist/drivers/memory.d.ts +0 -64
- package/dist/testing/fuzz-core.d.ts +0 -15
|
@@ -15,11 +15,6 @@ export declare function drainRegistry(registry: IRegistryDriver, options: {
|
|
|
15
15
|
namespace?: string;
|
|
16
16
|
maxScanSegments: number;
|
|
17
17
|
op: string;
|
|
18
|
-
/**
|
|
19
|
-
* How a caller of `op` raises the ceiling, completing "Narrow it with `namespace`, or …" in the refusal.
|
|
20
|
-
* Defaults to raising `maxScanSegments`, which is right wherever `op` takes that option itself.
|
|
21
|
-
*/
|
|
22
|
-
raise?: string;
|
|
23
18
|
/**
|
|
24
19
|
* Whether the caller could scope the scan down to a `namespace`. Defaults to true; `eraseNamespace` is already
|
|
25
20
|
* one namespace, so it passes false and the refusal does not tell it to narrow what it cannot.
|
|
@@ -2,8 +2,24 @@ import { type IAuditSink } from './audit.js';
|
|
|
2
2
|
import type { DropDeps, DropResult } from './erasure.js';
|
|
3
3
|
/** Default cap on retirements per sweep — a bounded batch, so a policy mistake costs one batch, not the fleet. */
|
|
4
4
|
export declare const DEFAULT_RETIRE_LIMIT = 100;
|
|
5
|
+
/**
|
|
6
|
+
* The shortest `tombstoneGraceMs` a sweep takes: one hour. A first load whose create got no answer is sent again, and a
|
|
7
|
+
* tombstone purged before that copy arrives lets it create the row again over the purged name, wrapped key included.
|
|
8
|
+
* A grace of seconds is for a test, which advances an injected clock.
|
|
9
|
+
*/
|
|
10
|
+
export declare const MIN_TOMBSTONE_GRACE_MS = 3600000;
|
|
11
|
+
/**
|
|
12
|
+
* Options of a retention sweep: the instant to judge policies at (`now`), which segments it considers, how many it
|
|
13
|
+
* retires in one call, how it finds them, whether it only reports (`dryRun`), where retirements are audited (`audit`),
|
|
14
|
+
* and whether and when it purges the tombstones of earlier retirements.
|
|
15
|
+
*/
|
|
5
16
|
export interface RetireExpiredOptions {
|
|
6
|
-
/**
|
|
17
|
+
/**
|
|
18
|
+
* Scope the sweep to one namespace. Omit to sweep every namespace the registry knows, segments loaded with no
|
|
19
|
+
* namespace included. `'_default'`, the label the export tool and the disaster-recovery samples print, and the
|
|
20
|
+
* bucket's object keys use, for those segments, is an ordinary namespace name here: it scopes the sweep to a
|
|
21
|
+
* namespace of that name, not to them.
|
|
22
|
+
*/
|
|
7
23
|
readonly namespace?: string;
|
|
8
24
|
/**
|
|
9
25
|
* The instant to compare policies against (epoch-ms). Required here because `core/` takes its time from the
|
|
@@ -11,7 +27,7 @@ export interface RetireExpiredOptions {
|
|
|
11
27
|
*/
|
|
12
28
|
readonly now: number;
|
|
13
29
|
/**
|
|
14
|
-
* Maximum segments to retire in this cycle (default 100). It also bounds the purges attempted (a purge the registry
|
|
30
|
+
* Maximum segments to retire in this cycle (default 100; an integer of at least 1, `null` refused). It also bounds the purges attempted (a purge the registry
|
|
15
31
|
* refuses is not charged; {@link RetireExpiredResult.purgeFaults} says how purging stops after refusals), the
|
|
16
32
|
* due-index pointers to nothing one call removes, and, counted apart from the retirements, the tombstones this sweep
|
|
17
33
|
* did not write whose pointers one call removes.
|
|
@@ -21,7 +37,10 @@ export interface RetireExpiredOptions {
|
|
|
21
37
|
readonly dryRun?: boolean;
|
|
22
38
|
/** Forwarded to each `dropSegment`, so every retirement lands in the audit trail as `segment.dispose`. */
|
|
23
39
|
readonly audit?: IAuditSink;
|
|
24
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Ceiling on rows enumerated (default 250,000); exceeding it throws. On an `'index'` scan the check runs before
|
|
42
|
+
* each pointer's row is read, so live rows equal to the ceiling followed by a pointer whose row is gone throw too.
|
|
43
|
+
*/
|
|
25
44
|
readonly maxScanSegments?: number;
|
|
26
45
|
/**
|
|
27
46
|
* **Where the candidates come from.**
|
|
@@ -41,6 +60,8 @@ export interface RetireExpiredOptions {
|
|
|
41
60
|
* tombstone under the day the tombstone's grace ends, and the scan reads it back with the expiry pointers. A pointer
|
|
42
61
|
* whose segment is gone is removed once read, by an index scan in the buckets it reads, and by an unscoped `'fleet'`
|
|
43
62
|
* scan (no `namespace`) in every bucket.
|
|
63
|
+
*
|
|
64
|
+
* Any other value, a misspelt `'index'` or a `null` included, throws `ValidationError` rather than running the fleet scan.
|
|
44
65
|
*/
|
|
45
66
|
readonly scan?: 'fleet' | 'index';
|
|
46
67
|
/**
|
|
@@ -49,14 +70,17 @@ export interface RetireExpiredOptions {
|
|
|
49
70
|
* across restarts.
|
|
50
71
|
*/
|
|
51
72
|
readonly shards?: readonly number[];
|
|
52
|
-
/**
|
|
73
|
+
/**
|
|
74
|
+
* Total shards the fleet is split into. Required with {@link shards}; ignored without it. `shards` must list at least
|
|
75
|
+
* one number from 0 to `totalShards - 1`, each once, else `ValidationError`.
|
|
76
|
+
*/
|
|
53
77
|
readonly totalShards?: number;
|
|
54
78
|
/**
|
|
55
79
|
* How many **past** buckets an `'index'` scan reads besides the current one (default 7, one per day).
|
|
56
80
|
* A sweep that did not run — scaled to zero, a failed deploy, a paused
|
|
57
81
|
* schedule — leaves its buckets behind, and this is how far back a later cycle reaches for them. Bounded so a
|
|
58
82
|
* long outage costs a bounded number of list calls per cycle rather than one per day since the epoch;
|
|
59
|
-
* anything older is the `'fleet'` repair pass's job.
|
|
83
|
+
* anything older is the `'fleet'` repair pass's job. An integer from 0 to 3660 on either scan, else `ValidationError`.
|
|
60
84
|
*/
|
|
61
85
|
readonly lookbackBuckets?: number;
|
|
62
86
|
/**
|
|
@@ -76,7 +100,13 @@ export interface RetireExpiredOptions {
|
|
|
76
100
|
* zero is the opposite of another's is a reading hazard for whoever tunes both.
|
|
77
101
|
*/
|
|
78
102
|
readonly purgeTombstones?: boolean;
|
|
79
|
-
/**
|
|
103
|
+
/**
|
|
104
|
+
* How long a retirement's tombstone must age before this sweep deletes it: at least one hour (`ValidationError`
|
|
105
|
+
* below), 24 h by default. The floor keeps the tombstone in place longer than a first load's create can take to be
|
|
106
|
+
* sent again after it got no answer, which would otherwise create the row again over the purged name, wrapped key
|
|
107
|
+
* included. Keep it well above the longest time a first load's registry write can still arrive (seconds on S3 and
|
|
108
|
+
* GCS, over a minute on Azure Blob at the SDK's default retry schedule). `null` is refused.
|
|
109
|
+
*/
|
|
80
110
|
readonly tombstoneGraceMs?: number;
|
|
81
111
|
}
|
|
82
112
|
/** What happened to one segment in a sweep. */
|
|
@@ -88,10 +118,13 @@ export type RetireEntry = {
|
|
|
88
118
|
readonly expiresAt: number;
|
|
89
119
|
readonly result: DropResult;
|
|
90
120
|
/**
|
|
91
|
-
* Present when the
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
121
|
+
* Present when the **tombstone landed and the retirement then faulted**: its storage sweep or listing threw, or
|
|
122
|
+
* the shred's own write got no answer that a read of the row could settle (then no `segment.erase` was emitted:
|
|
123
|
+
* record the crypto-shred yourself). `result.generationsRemaining` lists what a listing of the bucket found after
|
|
124
|
+
* the fault (the generations the row named, when that listing failed too). A sweep's re-run does not collect
|
|
125
|
+
* those objects inside `tombstoneGraceMs`: run `store.dropSegment(ref, { confirmSegment })`. Reported as `retired`
|
|
126
|
+
* rather than `skipped` because the segment really is retired, and saying otherwise is the one thing a caller
|
|
127
|
+
* cannot recover from.
|
|
95
128
|
*/
|
|
96
129
|
readonly fault?: `failed: ${string}`;
|
|
97
130
|
} | {
|
|
@@ -114,7 +147,8 @@ export type RetireEntry = {
|
|
|
114
147
|
* `'invalid-policy'` — the row has an `expiresAt` that is not usable (hand-edited, or restored from another
|
|
115
148
|
* schema). Reported rather than ignored: reading as "never expires" on a segment someone believes is
|
|
116
149
|
* expiring is the silence that costs a retention commitment.
|
|
117
|
-
* `'limit'` —
|
|
150
|
+
* `'limit'` — never written: a segment the cycle's `limit` deferred gets no entry, and
|
|
151
|
+
* {@link RetireExpiredResult.limited} is what says the cap cut the pass short. Re-run to continue.
|
|
118
152
|
* `'tombstone-not-empty'` — a tombstone whose Storage generations are not gone even after a GC
|
|
119
153
|
* attempt, so its row is kept, and, unless the collection lost a race to a write of the row, a refused purge
|
|
120
154
|
* (see {@link RetireExpiredResult.purgeFaults}): the row is what keeps the segment reachable by the generation collection and
|
|
@@ -127,9 +161,14 @@ export type RetireEntry = {
|
|
|
127
161
|
* re-reads before every deletion precisely so cancelling an expiry works on a sweep already in flight.
|
|
128
162
|
* `` `failed: …` `` — any fault, isolated to this segment. A fault that happened *after* the tombstone
|
|
129
163
|
* landed is reported as `retired` with a `fault` instead, because that segment IS retired.
|
|
164
|
+
* When the row cannot be read afterwards either, the reason says the tombstone may have landed.
|
|
130
165
|
*/
|
|
131
166
|
readonly reason: 'invalid-policy' | 'limit' | 'tombstone-not-empty' | 'policy-changed' | `failed: ${string}`;
|
|
132
167
|
};
|
|
168
|
+
/**
|
|
169
|
+
* What a retention sweep did: the rows it read, what it retired, and a ledger entry for each segment it acted on or
|
|
170
|
+
* skipped.
|
|
171
|
+
*/
|
|
133
172
|
export interface RetireExpiredResult {
|
|
134
173
|
/** Registry rows enumerated. */
|
|
135
174
|
readonly scanned: number;
|
|
@@ -146,11 +185,15 @@ export interface RetireExpiredResult {
|
|
|
146
185
|
readonly retired: number;
|
|
147
186
|
/** Segments a real sweep **would** have retired. Only ever non-zero under `dryRun`. */
|
|
148
187
|
readonly wouldRetire: number;
|
|
149
|
-
/**
|
|
188
|
+
/**
|
|
189
|
+
* Tombstone rows this sweep saw deleted. Zero under `dryRun`; the `would-purge-tombstone` entries carry the preview.
|
|
190
|
+
* A purge another sweep made at the same moment is counted by both.
|
|
191
|
+
*/
|
|
150
192
|
readonly tombstonesPurged: number;
|
|
151
193
|
/**
|
|
152
|
-
* True when `limit` cut the
|
|
153
|
-
* of tombstones this sweep did not write leaves
|
|
194
|
+
* True when `limit` cut the pass short — **more work remains**: retirements, or purges of this sweep's tombstones,
|
|
195
|
+
* which the same `limit` counts. Re-run. The cap on removing pointers of tombstones this sweep did not write leaves
|
|
196
|
+
* it false.
|
|
154
197
|
*/
|
|
155
198
|
readonly limited: boolean;
|
|
156
199
|
/**
|
|
@@ -163,7 +206,7 @@ export interface RetireExpiredResult {
|
|
|
163
206
|
*
|
|
164
207
|
* A refused purge is not charged to `limit`, so retirements go on: a tombstone that cannot be purged does not hold
|
|
165
208
|
* the segments behind it past their expiry. **Purging stops for the rest of the call after
|
|
166
|
-
*
|
|
209
|
+
* 3 refused purges in a row**, and a purge that succeeds starts the count
|
|
167
210
|
* again. A blanket refusal therefore costs three attempts a call, and a refusal particular to one row (a blob with a
|
|
168
211
|
* snapshot, an object under a legal hold) holds nothing behind it. The next call tries again. Check this field; a
|
|
169
212
|
* ledger entry for each is `skipped`, but a caller that reads only `retired` sees none of it.
|
|
@@ -171,8 +214,10 @@ export interface RetireExpiredResult {
|
|
|
171
214
|
readonly purgeFaults: number;
|
|
172
215
|
/** The first of those faults, as a ledger reason (`failed: …` with the provider's message). Absent when there were none. */
|
|
173
216
|
readonly firstPurgeFault?: `failed: ${string}`;
|
|
217
|
+
/** Whether the sweep was a dry run, which changes nothing. */
|
|
174
218
|
readonly dryRun: boolean;
|
|
175
|
-
/** Per-segment ledger. Inspect it: a `skipped` or failed entry is a segment that still holds data
|
|
219
|
+
/** Per-segment ledger. Inspect it: a `skipped` or failed entry is a segment that still holds data, except one whose reason says the row could
|
|
220
|
+
* not be read afterwards: its tombstone may have landed. */
|
|
176
221
|
readonly entries: readonly RetireEntry[];
|
|
177
222
|
}
|
|
178
223
|
/**
|
package/dist/core/retention.d.ts
CHANGED
|
@@ -22,15 +22,23 @@ export interface RetentionPolicy {
|
|
|
22
22
|
export interface RetentionDeps extends ChurnDeps {
|
|
23
23
|
readonly registry: IRegistryDriver;
|
|
24
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* What setting a segment's retention policy did: the policy now stored, and whether it created the row and wrote the
|
|
27
|
+
* due-index pointer.
|
|
28
|
+
*/
|
|
25
29
|
export interface SetRetentionResult {
|
|
30
|
+
/** The segment's name. */
|
|
26
31
|
readonly segment: string;
|
|
32
|
+
/** The segment's namespace, or `undefined` for the default one. */
|
|
27
33
|
readonly namespace?: string;
|
|
28
34
|
/** The policy now stored on the row. */
|
|
29
35
|
readonly expiresAt: number;
|
|
30
36
|
/**
|
|
31
37
|
* True iff this call **minted the registry row** — a policy recorded before the segment's first load. The
|
|
32
38
|
* segment is now enumerable by `registry.list()` and therefore by every fleet-wide operation; the row claims no
|
|
33
|
-
* Storage generation (`currentGen: null`), so reads answer empty until the first load publishes onto it.
|
|
39
|
+
* Storage generation (`currentGen: null`), so reads answer empty until the first load publishes onto it. A create
|
|
40
|
+
* that got no answer is read back and counts when the row carries this call's write token: a later policy or lease
|
|
41
|
+
* write does not hide it, and a later load's publish does (the call then decides again, and answers `false`).
|
|
34
42
|
*/
|
|
35
43
|
readonly createdRow: boolean;
|
|
36
44
|
/**
|
|
@@ -59,7 +67,7 @@ export declare function withoutSweepMark(meta: GovernanceMeta | undefined): Gove
|
|
|
59
67
|
* The three-way answer exists so a fleet sweep can *report* a malformed policy instead of choosing between two
|
|
60
68
|
* bad options: treating it as "no policy" hides an operator error on a segment someone believes is expiring, and
|
|
61
69
|
* throwing aborts a ledger over one bad row. Nothing writes an invalid policy through
|
|
62
|
-
*
|
|
70
|
+
* `store.setRetention` — hand-edited rows, an older/newer writer, or a restore are how it happens.
|
|
63
71
|
*/
|
|
64
72
|
export declare function readRetentionPolicy(meta: GovernanceMeta | undefined): RetentionPolicy | null | 'invalid';
|
|
65
73
|
/** The stored policy for one segment, or `null` if it has none (and `'invalid'` for a malformed one). */
|
|
@@ -70,14 +78,17 @@ export declare function getSegmentRetention(ref: SegmentRef, deps: RetentionDeps
|
|
|
70
78
|
* Idempotent and safe to re-run: writing the same instant twice is a no-op in effect. Other keys already in the
|
|
71
79
|
* row's `retention` metadata are preserved — this owns one key, not the whole object.
|
|
72
80
|
*
|
|
73
|
-
* Refuses a `destroyed` segment: a
|
|
74
|
-
* a
|
|
81
|
+
* Refuses a `destroyed` segment: a sweep never retires a tombstone (it purges only its own, by their stamp), so a
|
|
82
|
+
* policy on one would never act. Objects a destroy left in the bucket are deleted by `dropSegment` on the tombstone,
|
|
83
|
+
* not by a policy.
|
|
75
84
|
*/
|
|
76
85
|
export declare function setSegmentRetention(ref: SegmentRef, deps: RetentionDeps, policy: RetentionPolicy): Promise<SetRetentionResult>;
|
|
77
86
|
/**
|
|
78
|
-
* Remove a segment's expiry so no sweep will retire it. Returns whether
|
|
79
|
-
*
|
|
80
|
-
*
|
|
87
|
+
* Remove a segment's expiry so no sweep will retire it. Returns whether this call removed a policy (`false` for a
|
|
88
|
+
* segment with no row, or one that had no policy, and also when a write that got no answer cannot be shown to be this
|
|
89
|
+
* call's and the row holds no expiry: the expiry is gone either way, and its due-index pointer is removed. A later
|
|
90
|
+
* write that set an expiry is cleared too, and the answer is `true`). A separate verb from setting one, because "cancel
|
|
91
|
+
* the expiry" and "expire at time T" are different intents and a magic sentinel value for the first is how a typo
|
|
81
92
|
* becomes a deletion.
|
|
82
93
|
*
|
|
83
94
|
* Other `retention` keys are left in place; if removing the expiry empties the metadata, the field is cleared.
|
package/dist/core/retry.d.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import type { Clock, Rng } from './determinism.js';
|
|
2
|
+
/**
|
|
3
|
+
* How the store retries a transient read: attempts, exponential backoff with a cap, and jitter. By default 4 attempts,
|
|
4
|
+
* a 50 ms base delay, ×2, a 2 s cap and full jitter; the store's `retry` option overrides them a field at a time.
|
|
5
|
+
*/
|
|
2
6
|
export interface RetryPolicy {
|
|
3
7
|
/** Total attempts including the first (so `maxAttempts: 4` = 1 try + 3 retries). Must be ≥ 1. */
|
|
4
8
|
readonly maxAttempts: number;
|
|
@@ -44,7 +48,7 @@ export declare function backoffDelayMs(policy: RetryPolicy, attempt: number): nu
|
|
|
44
48
|
/** Apply the policy's jitter to a computed delay using the injected RNG. */
|
|
45
49
|
export declare function applyJitter(policy: RetryPolicy, delayMs: number, rng: Rng): number;
|
|
46
50
|
/** A clock that can wait: the one member {@link jitteredWait} uses. */
|
|
47
|
-
interface Sleeper {
|
|
51
|
+
export interface Sleeper {
|
|
48
52
|
readonly sleep?: ((ms: number) => Promise<void>) | undefined;
|
|
49
53
|
}
|
|
50
54
|
/**
|
|
@@ -79,4 +83,3 @@ export declare function checkedRetryPolicy(policy: RetryPolicy): RetryPolicy;
|
|
|
79
83
|
* the *operation's* error — never a wrapper — so callers keep their typed-error branching.
|
|
80
84
|
*/
|
|
81
85
|
export declare function withRetry<T>(op: () => Promise<T>, policy: RetryPolicy, deps: RetryDeps): Promise<T>;
|
|
82
|
-
export {};
|
package/dist/core/rollback.d.ts
CHANGED
|
@@ -17,8 +17,12 @@ export interface GenerationListDeps extends ChurnDeps {
|
|
|
17
17
|
}
|
|
18
18
|
/** One generation present in the bucket. */
|
|
19
19
|
export interface GenerationEntry {
|
|
20
|
+
/** The generation's number. */
|
|
20
21
|
readonly generation: number;
|
|
21
|
-
/**
|
|
22
|
+
/**
|
|
23
|
+
* Whether the registry pointer currently names this generation. `false` for every generation of a destroyed
|
|
24
|
+
* segment: a destroyed row keeps the pointer it had, but no read resolves a generation through it.
|
|
25
|
+
*/
|
|
22
26
|
readonly current: boolean;
|
|
23
27
|
/**
|
|
24
28
|
* Whether the row's record of the generations it published, `knownGens`, names this generation. A rollback onto a known
|
|
@@ -51,6 +55,7 @@ export interface GenerationEntry {
|
|
|
51
55
|
* carries the id count and metadata its row records, from the same read.
|
|
52
56
|
*/
|
|
53
57
|
export declare function listGenerations(ref: SegmentRef, deps: GenerationListDeps): Promise<GenerationEntry[]>;
|
|
58
|
+
/** What a rollback moved: the generation the pointer named before, and the one it names now. */
|
|
54
59
|
export interface RollbackResult {
|
|
55
60
|
/** The generation the pointer named before the rollback. `null` when the segment had no current generation. */
|
|
56
61
|
readonly fromGeneration: number | null;
|
|
@@ -75,8 +80,8 @@ export interface RollbackResult {
|
|
|
75
80
|
* - a generation whose object is **not in the bucket** — collected, or never written — is refused with
|
|
76
81
|
* `NotFoundError` rather than pointed at, because a pointer naming a missing object is the one state the whole
|
|
77
82
|
* design exists to avoid;
|
|
78
|
-
* - a **crypto-shredded
|
|
79
|
-
*
|
|
83
|
+
* - a **destroyed** segment (crypto-shredded, destroyed in the clear or dropped) is refused: no generation resolves
|
|
84
|
+
* through a destroyed row, so moving its pointer would change nothing a reader sees;
|
|
80
85
|
* - a target that is **not what the row says the segment is** is refused with `IntegrityError`: a cleartext object
|
|
81
86
|
* under a row with keys, or an encrypted one under a row with none, either of which every read would then refuse.
|
|
82
87
|
* One tail read of the target decides it, as does a footer that fails its own checks;
|
|
@@ -109,15 +114,21 @@ export interface RollbackResult {
|
|
|
109
114
|
* failure of the keystore is thrown. After the swap the target's footer is read once more, and the pointer put back,
|
|
110
115
|
* restoring the summary and the record the row had, unless it is still the object that was read: collection can delete a
|
|
111
116
|
* target below the old pointer between the read and the swap, and an erasure can delete one above it and a load take its
|
|
112
|
-
* number. A swap that gets no answer is settled by reading the row
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
117
|
+
* number. A swap that meets a conflict or gets no answer is settled by reading the row. On a registry that reports its
|
|
118
|
+
* writes' tokens, one the row shows landed, its pointer at the target over the token of a swap this call sent, is
|
|
119
|
+
* checked as any swap is, the move audited and reported, whatever else was written to the row since (a lease, a
|
|
120
|
+
* retention or residency write, or a write of the ledger alone). Otherwise the call throws, and nothing is audited or
|
|
121
|
+
* deleted: the `WriteConflictError` after a conflict, and the `TransientError` after no answer, since it may have
|
|
122
|
+
* landed or may still land and a re-run finds the pointer where it is. Another writer's write of the same pointer is not
|
|
123
|
+
* taken for it, another rollback's identical move included, and neither is a write that renewed the row's `pointerId` over
|
|
124
|
+
* this swap, which cannot be told from it. A registry that reports no tokens settles a swap that got no answer by the
|
|
125
|
+
* row it would leave, and never settles a conflict as landed. Over such a registry a write by another writer is not
|
|
126
|
+
* taken for the swap unless it leaves the very row this swap would. On a cleartext segment two writes can: another
|
|
127
|
+
* rollback onto the same target, whatever row it read, and a load that wrote under that number the very object the
|
|
128
|
+
* rollback read and carried the ledger the rollback would (a load that carries a usable ledger, as one with a `keep`
|
|
129
|
+
* above 64 does; over a row with no usable ledger the rollback writes none and a load seeds one, so those rows differ).
|
|
130
|
+
* On an encrypted segment a rollback with no key at hand writes no summary, so another such rollback onto the same
|
|
131
|
+
* target can too. That call then reports and audits the move as its own; the pointer is where it asked for it.
|
|
121
132
|
*
|
|
122
133
|
* What it does **not** do is delete anything. The generations above the new pointer stay in the bucket, which is
|
|
123
134
|
* what makes the rollback reversible — roll forward again by naming one of them. They are also then *above*
|
package/dist/core/validate.d.ts
CHANGED
|
@@ -16,6 +16,19 @@ export declare function isWellFormedString(value: string): boolean;
|
|
|
16
16
|
* same way, and a second copy of that rule is how the two drifted apart before.
|
|
17
17
|
*/
|
|
18
18
|
export declare function encodedNameLength(value: string): number;
|
|
19
|
+
/**
|
|
20
|
+
* The keys a segment ref takes: its address, and the two a `segments()` row also carries so a listing row can be
|
|
21
|
+
* passed straight back. `discover.ts` holds this list to `SegmentInfo` at compile time, so a field added there is a
|
|
22
|
+
* type error until it is listed here.
|
|
23
|
+
*/
|
|
24
|
+
export declare const REF_KEYS: readonly ["segment", "namespace", "currentGen", "status"];
|
|
25
|
+
/**
|
|
26
|
+
* Throws `ValidationError` unless `ref` is an object whose `segment` and, when given, `namespace` are valid names, and
|
|
27
|
+
* which holds no other key: a misspelt `namespace` would otherwise address the segment of that name in the default
|
|
28
|
+
* namespace. The keys it takes are a port's: `segment`, `namespace`, a `GenKey`'s `generation`, a `ChunkRef`'s
|
|
29
|
+
* `chunkKey`, and a registry row's own fields and a `segments()` row's, so a row a driver listed can be passed back.
|
|
30
|
+
* It does not refuse a reserved namespace, which a driver must be able to store.
|
|
31
|
+
*/
|
|
19
32
|
export declare function validateSegmentRef(ref: SegmentRef): void;
|
|
20
33
|
/**
|
|
21
34
|
* The namespace prefix the library keeps for its own bookkeeping: the due index stores one pointer row per
|
|
@@ -40,8 +53,26 @@ export declare function isReservedNamespace(namespace: string | undefined): bool
|
|
|
40
53
|
* rows live in the reserved namespace and go through the drivers.
|
|
41
54
|
*/
|
|
42
55
|
export declare function validateUserNamespace(namespace: string): void;
|
|
43
|
-
/**
|
|
44
|
-
|
|
56
|
+
/**
|
|
57
|
+
* The keys a driver port takes beyond a ref's own. The store hands its ports a generation key (a ref and a
|
|
58
|
+
* `generation`) or a chunk ref (a ref and a `chunkKey`) wherever they name a segment, and an operator repairing a
|
|
59
|
+
* registry hands them a row from `registry.list()` or `get()`, whose fields are fixed and listed here. A test holds
|
|
60
|
+
* the row fields to `RegistryRecord`'s.
|
|
61
|
+
*/
|
|
62
|
+
export declare const PORT_KEYS: readonly string[];
|
|
63
|
+
/**
|
|
64
|
+
* Refuse a segment ref that holds a key it does not take, naming each such key, and a value that is a segment
|
|
65
|
+
* handle or an array rather than a ref. A misspelt `namespace` would otherwise address the segment of that name in
|
|
66
|
+
* the default namespace, and a drop, a rollback or a crypto-shred would act on the wrong segment. Only the ref's own
|
|
67
|
+
* enumerable string keys are read, one whose value is `undefined` included. `verb` names the call in the message.
|
|
68
|
+
*/
|
|
69
|
+
export declare function refuseUnknownRefKeys(ref: object, verb: string, extraKeys?: readonly string[]): void;
|
|
70
|
+
/**
|
|
71
|
+
* Validate a segment ref an application hands in: the name rules, and a namespace outside the reserved prefix. With
|
|
72
|
+
* `verb`, the call a caller made, it also refuses a key a ref does not take, beyond `extraKeys`; leave `verb` out only
|
|
73
|
+
* for an internal key that extends a ref with keys of its own.
|
|
74
|
+
*/
|
|
75
|
+
export declare function validateUserRef(ref: SegmentRef, verb?: string, extraKeys?: readonly string[]): void;
|
|
45
76
|
/**
|
|
46
77
|
* Validate the keys of a `getChunks` stream: in ascending order, a key may repeat. Checked before anything is
|
|
47
78
|
* resolved or read, so a caller's mistake costs no request.
|
package/dist/driver-kit.d.ts
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
-
export type { GenKey, IRegistryDriver, IStorageDriver, SegmentRef, StorageBackend, StorageCaps, StorageDeleteOptions, TailRead, } from './core/ports.js';
|
|
1
|
+
export type { GenKey, IRegistryDriver, IStorageDriver, LifecycleVerdict, SegmentRef, StorageBackend, StorageCaps, StorageDeleteOptions, TailRead, } from './core/ports.js';
|
|
2
2
|
export { brandAsBackend, STORAGE_BACKEND } from './core/ports.js';
|
|
3
|
-
export type {
|
|
4
|
-
export {
|
|
3
|
+
export type { ClearRegistrySummary, GenerationMetadata, GovernanceMeta, KnownGeneration, LeaseEntry, NewRegistryRecord, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, RegistryWriteOptions, SealedRegistrySummary, Token, } from './core/ports.js';
|
|
4
|
+
export type { ChunkRead, ChunkRef, GenerationSummary, ReadChunksOptions, SegmentSize, StorageChunkSource, } from './core/ports.js';
|
|
5
|
+
export type { Aead, AeadSealed, IKeystore, WrappedDek } from './core/crypto.js';
|
|
6
|
+
export type { Entropy } from './core/determinism.js';
|
|
7
|
+
export type { IMetricsSink, MetricEvent, MetricOpName } from './core/metrics.js';
|
|
8
|
+
export { CloudRoaringError, errorKind, IntegrityError, NotFoundError, TransientError, unsettledConflict, ValidationError, WriteConflictError, } from './core/errors.js';
|
|
9
|
+
export type { ErrorKind } from './core/errors.js';
|
|
5
10
|
export { RESOLVED_FIELDS, pointerIdOf, renewPointer, renewsPointer } from './core/pointer-id.js';
|
|
6
11
|
export { validateSegmentRef } from './core/validate.js';
|
|
7
12
|
export type { BlobSink } from './core/blob.js';
|
|
@@ -10,6 +15,6 @@ export type { ObjectRegistryStore, ObjectRow } from './drivers/_shared/object-re
|
|
|
10
15
|
export { encodeNameForKey, encodeNameForPath, namespaceKeyPart, namespacePathPart, } from './drivers/_shared/keys.js';
|
|
11
16
|
export { normalizeObjectPrefix, prefixPart } from './drivers/_shared/object-registry-keys.js';
|
|
12
17
|
export { parseGenerationFromKey, segmentObjectPrefix, storageObjectKey, } from './drivers/_shared/object-keys.js';
|
|
13
|
-
export { refuseUnknownOptions, requireBool, requirePositiveInt } from './drivers/_shared/options.js';
|
|
18
|
+
export { checkDeleteOptions, checkRegistryWriteOptions, refuseUnknownOptions, requireBool, requireNonEmptyString, requirePositiveInt, } from './drivers/_shared/options.js';
|
|
14
19
|
export { isTransportFault, toTransient } from './drivers/_shared/faults.js';
|
|
15
20
|
export { concatBytes } from './drivers/_shared/bytes.js';
|
package/dist/driver-kit.js
CHANGED
|
@@ -1,5 +1,26 @@
|
|
|
1
|
+
import {
|
|
2
|
+
mapWithConcurrency
|
|
3
|
+
} from "./chunk-CGILDXGP.js";
|
|
1
4
|
import {
|
|
2
5
|
DEFAULT_NAMESPACE,
|
|
6
|
+
applyRegistryPatch,
|
|
7
|
+
checkDeleteOptions,
|
|
8
|
+
checkRegistryWriteOptions,
|
|
9
|
+
entropyIsAvailable,
|
|
10
|
+
namespaceKeyPart,
|
|
11
|
+
namespacePathPart,
|
|
12
|
+
newIncarnationToken,
|
|
13
|
+
nextRegistryToken,
|
|
14
|
+
parseRegistryEnvelope,
|
|
15
|
+
recordFromNew,
|
|
16
|
+
requirePositiveInt,
|
|
17
|
+
serializeRegistryEnvelope,
|
|
18
|
+
validateNewRegistryRecord,
|
|
19
|
+
validateRegistryPatch,
|
|
20
|
+
webCryptoEntropy
|
|
21
|
+
} from "./chunk-7GT632UF.js";
|
|
22
|
+
import {
|
|
23
|
+
CloudRoaringError,
|
|
3
24
|
IntegrityError,
|
|
4
25
|
NotFoundError,
|
|
5
26
|
RESOLVED_FIELDS,
|
|
@@ -7,37 +28,24 @@ import {
|
|
|
7
28
|
TransientError,
|
|
8
29
|
ValidationError,
|
|
9
30
|
WriteConflictError,
|
|
10
|
-
applyRegistryPatch,
|
|
11
31
|
brandAsBackend,
|
|
12
32
|
decodeNameFromKey,
|
|
13
33
|
encodeNameForKey,
|
|
14
34
|
encodeNameForPath,
|
|
15
|
-
|
|
16
|
-
isIntegrityError,
|
|
17
|
-
isNotFoundError,
|
|
35
|
+
errorKind,
|
|
18
36
|
isTransientError,
|
|
19
|
-
|
|
37
|
+
isUnsettledConflict,
|
|
20
38
|
isWriteConflictError,
|
|
21
|
-
mapWithConcurrency,
|
|
22
|
-
namespaceKeyPart,
|
|
23
|
-
namespacePathPart,
|
|
24
|
-
newIncarnationToken,
|
|
25
|
-
nextRegistryToken,
|
|
26
|
-
parseRegistryEnvelope,
|
|
27
39
|
pointerIdOf,
|
|
28
|
-
recordFromNew,
|
|
29
40
|
refuseUnknownOptions,
|
|
30
41
|
renewPointer,
|
|
31
42
|
renewsPointer,
|
|
32
43
|
requireBool,
|
|
33
44
|
requireInteger,
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
validateSegmentRef,
|
|
39
|
-
webCryptoEntropy
|
|
40
|
-
} from "./chunk-IGLIZBR7.js";
|
|
45
|
+
requireNonEmptyString,
|
|
46
|
+
unsettledConflict,
|
|
47
|
+
validateSegmentRef
|
|
48
|
+
} from "./chunk-MVUD4R72.js";
|
|
41
49
|
|
|
42
50
|
// src/drivers/_shared/object-registry-keys.ts
|
|
43
51
|
var REGISTRY_SUFFIX = ".reg";
|
|
@@ -52,6 +60,12 @@ function prefixPart(prefix) {
|
|
|
52
60
|
}
|
|
53
61
|
function normalizeObjectPrefix(prefix) {
|
|
54
62
|
if (prefix === void 0) return void 0;
|
|
63
|
+
const given = prefix;
|
|
64
|
+
if (typeof given !== "string") {
|
|
65
|
+
throw new ValidationError(
|
|
66
|
+
`prefix must be a string; got ${given === null ? "null" : Array.isArray(given) ? "array" : typeof given}`
|
|
67
|
+
);
|
|
68
|
+
}
|
|
55
69
|
for (const ch of prefix) {
|
|
56
70
|
const code = ch.charCodeAt(0);
|
|
57
71
|
if (code < 32 || code === 127) {
|
|
@@ -121,6 +135,7 @@ var MAX_DELETE_ATTEMPTS = 8;
|
|
|
121
135
|
var MAX_READ_ATTEMPTS = 8;
|
|
122
136
|
var LIST_READ_CONCURRENCY = 48;
|
|
123
137
|
var ObjectVersionRaced = class extends Error {
|
|
138
|
+
/** Builds the error naming the object whose pinned version vanished. */
|
|
124
139
|
constructor(objectKey) {
|
|
125
140
|
super(`registry object was overwritten mid-read: ${objectKey}`);
|
|
126
141
|
this.name = "ObjectVersionRaced";
|
|
@@ -143,6 +158,10 @@ var ObjectStoreRegistry = class {
|
|
|
143
158
|
prefix;
|
|
144
159
|
now;
|
|
145
160
|
entropy;
|
|
161
|
+
/**
|
|
162
|
+
* Strong reads, always. `canWrite` is `false` when the runtime has no source of random bytes for tokens, and
|
|
163
|
+
* `conditionalDelete` is `true` when the store deletes a row only while it is still the version read.
|
|
164
|
+
*/
|
|
146
165
|
capabilities() {
|
|
147
166
|
const conditionalDelete = this.removesRows();
|
|
148
167
|
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete } : { strongRead: true, canWrite: false, conditionalDelete };
|
|
@@ -164,7 +183,13 @@ var ObjectStoreRegistry = class {
|
|
|
164
183
|
this.observed.set(row.env.record, { key, bytes: row.bytes, version: row.version });
|
|
165
184
|
return row.env.record;
|
|
166
185
|
}
|
|
186
|
+
/**
|
|
187
|
+
* Write a new row, create-only over none or over a tombstone. `held: null` is the publish's contract: its caller read
|
|
188
|
+
* the row absent. A direct caller that passes it over a tombstone may see a conflict when the read-back after the
|
|
189
|
+
* lost create faults, though no live row exists.
|
|
190
|
+
*/
|
|
167
191
|
async create(ref, record, options) {
|
|
192
|
+
checkRegistryWriteOptions(options);
|
|
168
193
|
const checked = validateNewRegistryRecord(record);
|
|
169
194
|
const key = registryObjectKey(this.prefix, ref);
|
|
170
195
|
let lost;
|
|
@@ -176,9 +201,16 @@ var ObjectStoreRegistry = class {
|
|
|
176
201
|
lost = err;
|
|
177
202
|
}
|
|
178
203
|
}
|
|
179
|
-
|
|
204
|
+
let current;
|
|
205
|
+
try {
|
|
206
|
+
current = await this.readRow(key);
|
|
207
|
+
} catch (err) {
|
|
208
|
+
if (lost !== void 0 && isTransientError(err)) throw lost;
|
|
209
|
+
throw err;
|
|
210
|
+
}
|
|
180
211
|
if (current !== null && !current.env.deleted) {
|
|
181
|
-
|
|
212
|
+
const message = `registry row already exists for segment ${ref.segment}`;
|
|
213
|
+
throw isUnsettledConflict(lost) ? unsettledConflict(message) : new WriteConflictError(message);
|
|
182
214
|
}
|
|
183
215
|
if (lost !== void 0 && current === null) throw lost;
|
|
184
216
|
return this.createOver(key, ref, checked, current ?? void 0, options);
|
|
@@ -195,6 +227,7 @@ var ObjectStoreRegistry = class {
|
|
|
195
227
|
return { token };
|
|
196
228
|
}
|
|
197
229
|
async compareAndSwap(ref, expected, patch, options) {
|
|
230
|
+
checkRegistryWriteOptions(options);
|
|
198
231
|
const checked = validateRegistryPatch(patch);
|
|
199
232
|
const key = registryObjectKey(this.prefix, ref);
|
|
200
233
|
const current = this.observedRow(key, expected, options?.held) ?? await this.readRow(key);
|
|
@@ -334,8 +367,10 @@ function segmentObjectPrefix(prefix, ref) {
|
|
|
334
367
|
return `${prefixPart(prefix)}${namespaceKeyPart(ref.namespace)}/segments/${encodeNameForKey(ref.segment)}.`;
|
|
335
368
|
}
|
|
336
369
|
function storageObjectKey(prefix, key) {
|
|
370
|
+
validateSegmentRef(key);
|
|
337
371
|
requireInteger(key.generation, { min: 0, what: "generation" });
|
|
338
|
-
|
|
372
|
+
const ref = { segment: key.segment, namespace: key.namespace };
|
|
373
|
+
return `${segmentObjectPrefix(prefix, ref)}${key.generation}${SUFFIX}`;
|
|
339
374
|
}
|
|
340
375
|
function parseGenerationFromKey(segmentPrefix, objectKey) {
|
|
341
376
|
if (!objectKey.startsWith(segmentPrefix) || !objectKey.endsWith(SUFFIX)) return null;
|
|
@@ -378,6 +413,7 @@ function concatBytes(parts) {
|
|
|
378
413
|
return out;
|
|
379
414
|
}
|
|
380
415
|
export {
|
|
416
|
+
CloudRoaringError,
|
|
381
417
|
IntegrityError,
|
|
382
418
|
MAX_ROW_BYTES,
|
|
383
419
|
NotFoundError,
|
|
@@ -389,15 +425,13 @@ export {
|
|
|
389
425
|
ValidationError,
|
|
390
426
|
WriteConflictError,
|
|
391
427
|
brandAsBackend,
|
|
428
|
+
checkDeleteOptions,
|
|
429
|
+
checkRegistryWriteOptions,
|
|
392
430
|
concatBytes,
|
|
393
431
|
encodeNameForKey,
|
|
394
432
|
encodeNameForPath,
|
|
395
|
-
|
|
396
|
-
isNotFoundError,
|
|
397
|
-
isTransientError,
|
|
433
|
+
errorKind,
|
|
398
434
|
isTransportFault,
|
|
399
|
-
isValidationError,
|
|
400
|
-
isWriteConflictError,
|
|
401
435
|
namespaceKeyPart,
|
|
402
436
|
namespacePathPart,
|
|
403
437
|
normalizeObjectPrefix,
|
|
@@ -408,10 +442,12 @@ export {
|
|
|
408
442
|
renewPointer,
|
|
409
443
|
renewsPointer,
|
|
410
444
|
requireBool,
|
|
445
|
+
requireNonEmptyString,
|
|
411
446
|
requirePositiveInt,
|
|
412
447
|
segmentObjectPrefix,
|
|
413
448
|
storageObjectKey,
|
|
414
449
|
toTransient,
|
|
450
|
+
unsettledConflict,
|
|
415
451
|
validateSegmentRef
|
|
416
452
|
};
|
|
417
453
|
//# sourceMappingURL=driver-kit.js.map
|