@cloudbitmaps/core 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -5
- package/dist/{chunk-YSC7NJWV.js → chunk-OSU7W2Y3.js} +811 -164
- package/dist/chunk-OSU7W2Y3.js.map +7 -0
- package/dist/core/audit.d.ts +12 -2
- package/dist/core/bit-route.d.ts +9 -1
- package/dist/core/blob.d.ts +12 -1
- package/dist/core/codec.d.ts +1 -1
- package/dist/core/combine-expr.d.ts +10 -8
- package/dist/core/combine-feed.d.ts +12 -18
- package/dist/core/combine-many.d.ts +29 -22
- package/dist/core/concurrency.d.ts +8 -0
- package/dist/core/consistency.d.ts +69 -17
- package/dist/core/crbm/reader.d.ts +52 -6
- package/dist/core/crbm-storage-source.d.ts +65 -452
- package/dist/core/crypto.d.ts +6 -1
- package/dist/core/determinism.d.ts +2 -1
- package/dist/core/engine.d.ts +222 -33
- package/dist/core/erase-id.d.ts +60 -15
- package/dist/core/erasure.d.ts +5 -4
- package/dist/core/errors.d.ts +61 -11
- package/dist/core/leases.d.ts +4 -3
- package/dist/core/load.d.ts +58 -45
- package/dist/core/lru.d.ts +2 -15
- package/dist/core/metrics.d.ts +5 -3
- package/dist/core/pinned-storage-source.d.ts +4 -4
- package/dist/core/ports.d.ts +96 -22
- package/dist/core/registry-scan.d.ts +0 -7
- package/dist/core/retention-sweep.d.ts +70 -66
- package/dist/core/retention.d.ts +2 -2
- package/dist/core/retry.d.ts +15 -18
- package/dist/core/rollback.d.ts +61 -31
- package/dist/core/validate.d.ts +2 -0
- package/dist/driver-kit.d.ts +11 -2
- package/dist/driver-kit.js +84 -16
- package/dist/driver-kit.js.map +3 -3
- package/dist/drivers/_shared/bytes.d.ts +6 -0
- package/dist/drivers/_shared/faults.d.ts +15 -0
- package/dist/drivers/_shared/object-keys.d.ts +18 -0
- package/dist/drivers/_shared/object-registry-keys.d.ts +0 -2
- package/dist/drivers/_shared/object-registry.d.ts +2 -0
- package/dist/drivers/_shared/options.d.ts +19 -0
- package/dist/drivers/backends.d.ts +2 -0
- package/dist/drivers/crypto.d.ts +1 -1
- package/dist/export/index.d.ts +9 -8
- package/dist/index.d.ts +11 -10
- package/dist/index.js +4202 -2833
- package/dist/index.js.map +4 -4
- package/package.json +7 -2
- package/dist/chunk-YSC7NJWV.js.map +0 -7
- package/dist/core/chunk-checks.d.ts +0 -34
- package/dist/core/chunk-stream.d.ts +0 -22
- package/dist/core/chunk-window.d.ts +0 -30
- package/dist/core/combine-ledger.d.ts +0 -43
- package/dist/core/crbm/crc32c.d.ts +0 -10
- package/dist/core/crbm/fingerprint.d.ts +0 -18
- package/dist/core/crbm/format.d.ts +0 -151
- package/dist/core/crbm/frame.d.ts +0 -10
- package/dist/core/crbm/plan-reads.d.ts +0 -44
- package/dist/core/crbm/varint.d.ts +0 -13
- package/dist/core/crbm/writer.d.ts +0 -68
- package/dist/core/due-index.d.ts +0 -53
- package/dist/core/generation-gc.d.ts +0 -179
- package/dist/core/item-pull.d.ts +0 -29
- package/dist/core/kept-generations.d.ts +0 -40
- package/dist/core/metadata.d.ts +0 -42
- package/dist/core/reader-defaults.d.ts +0 -29
- package/dist/core/summary.d.ts +0 -78
- package/dist/core/token.d.ts +0 -41
- package/dist/drivers/_shared/entropy.d.ts +0 -5
- package/dist/drivers/_shared/registry.d.ts +0 -104
- package/dist/drivers/localfs/exact-case.d.ts +0 -37
- package/dist/drivers/localfs/fs-util.d.ts +0 -33
- package/dist/drivers/localfs/paths.d.ts +0 -40
- package/dist/drivers/localfs/registry.d.ts +0 -44
- package/dist/drivers/localfs/storage.d.ts +0 -21
- package/dist/drivers/memory.d.ts +0 -61
- package/dist/testing/fuzz-core.d.ts +0 -15
|
@@ -4,8 +4,6 @@ import type { IRegistryDriver, RegistryRecord } from './ports.js';
|
|
|
4
4
|
* the fleet really is that large *and* the memory is there; narrow the scan with a `namespace` otherwise.
|
|
5
5
|
*/
|
|
6
6
|
export declare const DEFAULT_MAX_SCAN_SEGMENTS = 250000;
|
|
7
|
-
/** Fail fast on a bad ceiling BEFORE the (possibly huge) scan, not after. */
|
|
8
|
-
export declare function validateMaxScanSegments(value: number, op: string): void;
|
|
9
7
|
/**
|
|
10
8
|
* Drain every registry record (optionally one namespace) into an array, refusing past `maxScanSegments`.
|
|
11
9
|
*
|
|
@@ -17,11 +15,6 @@ export declare function drainRegistry(registry: IRegistryDriver, options: {
|
|
|
17
15
|
namespace?: string;
|
|
18
16
|
maxScanSegments: number;
|
|
19
17
|
op: string;
|
|
20
|
-
/**
|
|
21
|
-
* How a caller of `op` raises the ceiling, completing "Narrow it with `namespace`, or …" in the refusal.
|
|
22
|
-
* Defaults to raising `maxScanSegments`, which is right wherever `op` takes that option itself.
|
|
23
|
-
*/
|
|
24
|
-
raise?: string;
|
|
25
18
|
/**
|
|
26
19
|
* Whether the caller could scope the scan down to a `namespace`. Defaults to true; `eraseNamespace` is already
|
|
27
20
|
* one namespace, so it passes false and the refusal does not tell it to narrow what it cannot.
|
|
@@ -1,69 +1,7 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The retention **sweep** — the thing that acts on the policies `setSegmentRetention` records.
|
|
3
|
-
*
|
|
4
|
-
* `retireExpired` enumerates the registry, selects the segments whose `expiresAt` has passed, and retires each
|
|
5
|
-
* one through {@link dropSegment}. It deliberately **delegates rather than reimplements**: the registry → Storage
|
|
6
|
-
* ordering, the re-sweep for an object a load was still writing, and the `generationsRemaining` report are all
|
|
7
|
-
* load-bearing and already live there. A sweep that open-coded the deletions would be a second
|
|
8
|
-
* implementation of the most dangerous ordering in the library.
|
|
9
|
-
*
|
|
10
|
-
* **This is a call, not a daemon.** Nothing here schedules itself. You run it from whatever heartbeat your
|
|
11
|
-
* deployment already has — an EventBridge rule, a Kubernetes CronJob, a queue consumer, the job that runs your
|
|
12
|
-
* loads — and the library stays a library: it has to behave identically in a Lambda, an edge isolate and a
|
|
13
|
-
* long-lived server, and a timer that only works in one of those is worse than none.
|
|
14
|
-
*
|
|
15
|
-
* Three properties make it safe to point at a fleet:
|
|
16
|
-
*
|
|
17
|
-
* - **`dryRun` at the sweep level.** `dropSegment`'s `confirmSegment` guard is vacuous in a loop (it is the same
|
|
18
|
-
* variable twice), so the meaningful preview is here: it reports what it *would* retire, per segment, and
|
|
19
|
-
* touches nothing.
|
|
20
|
-
* - **A per-cycle `limit`.** Clock skew, a bad backfill, or a namespace-wide policy mistake should cost one
|
|
21
|
-
* bounded batch, not the fleet. When the limit bites, the result says so (`limited: true`) rather than looking
|
|
22
|
-
* like a completed sweep.
|
|
23
|
-
* - **A ledger, never an exception.** A per-segment fault is recorded and the sweep continues — the caller's
|
|
24
|
-
* load-bearing question is *which* segments were retired, and a throw from the middle of the loop answers it
|
|
25
|
-
* for none of them while having already retired some. Same posture as `eraseNamespace`.
|
|
26
|
-
*
|
|
27
|
-
* It also **purges the tombstones its own retirements leave behind**. A retired segment that had a registry row
|
|
28
|
-
* gets a `destroyed` row, and one dead row per retired daily bucket — or per retired dedup wave — is exactly the
|
|
29
|
-
* registry litter `dropSegment` already refuses to create for a row-less accumulator. Purging is narrow on
|
|
30
|
-
* purpose: only a tombstone carrying the **stamp this sweep writes on its own retirements** (so it is attributably
|
|
31
|
-
* ours: a crypto-shred's tombstone does not carry it, unless the shred lands between this sweep's re-read of the
|
|
32
|
-
* row and its drop, which then reports the tombstone dropped and stamps it), only after a grace period, and only
|
|
33
|
-
* once Storage is provably empty for it —
|
|
34
|
-
* because deleting the row is what makes the name reusable and takes the segment out of reach of
|
|
35
|
-
* the generation collection.
|
|
36
|
-
*
|
|
37
|
-
* **On a registry that reports `conditionalDelete`, the purge removes the row for good**, so a full scan stops paying a
|
|
38
|
-
* read for every name a namespace ever held: the scan costs what is live and what is inside its grace. The delete is
|
|
39
|
-
* fenced on the token the purge judged and applied by the store only to that version of the row. Each retirement
|
|
40
|
-
* files a pointer in the due index under the day its tombstone's grace ends, so `scan: 'index'` purges as well as
|
|
41
|
-
* retires; nothing on the row records that day, which the purge derives from the stamp. The purge removes every pointer
|
|
42
|
-
* it read to the row, and a pointer that outlives its row anyway (a purge that ran with another grace, a delete that
|
|
43
|
-
* landed and lost its response, a pointer older than an index scan's lookback) is removed by the next scan that reads
|
|
44
|
-
* it: an index scan in its buckets, and an unscoped fleet scan, which reads every pointer, in all of them. Where a
|
|
45
|
-
* delete only rewrites the row as a tombstone, none of this is done: nothing is removed for good, so a pointer would
|
|
46
|
-
* only add a row for every scan to read.
|
|
47
|
-
*/
|
|
48
1
|
import { type IAuditSink } from './audit.js';
|
|
49
2
|
import type { DropDeps, DropResult } from './erasure.js';
|
|
50
3
|
/** Default cap on retirements per sweep — a bounded batch, so a policy mistake costs one batch, not the fleet. */
|
|
51
4
|
export declare const DEFAULT_RETIRE_LIMIT = 100;
|
|
52
|
-
/**
|
|
53
|
-
* How many purges in a row the registry may refuse in one call before the sweep stops purging for the rest of it. A
|
|
54
|
-
* blanket refusal (a policy that denies delete) costs this many attempts and no more; a refusal particular to one row
|
|
55
|
-
* (a blob with a snapshot, an object under a legal hold) does not hold the purges behind it, since a purge that succeeds
|
|
56
|
-
* starts the count again.
|
|
57
|
-
*/
|
|
58
|
-
export declare const MAX_CONSECUTIVE_PURGE_FAULTS = 3;
|
|
59
|
-
/**
|
|
60
|
-
* Default delay before a retirement's own tombstone row is purged: 24 h.
|
|
61
|
-
*
|
|
62
|
-
* The row is a fence — while it exists, a load refuses the segment, so
|
|
63
|
-
* a load that was mid-write when the drop landed cannot resurrect it. That window is
|
|
64
|
-
* seconds to minutes in practice; a day of margin costs one tiny row and removes any need to reason about it.
|
|
65
|
-
*/
|
|
66
|
-
export declare const DEFAULT_TOMBSTONE_GRACE_MS = 86400000;
|
|
67
5
|
export interface RetireExpiredOptions {
|
|
68
6
|
/** Scope the sweep to one namespace. Omit to sweep every namespace the registry knows. */
|
|
69
7
|
readonly namespace?: string;
|
|
@@ -74,8 +12,9 @@ export interface RetireExpiredOptions {
|
|
|
74
12
|
readonly now: number;
|
|
75
13
|
/**
|
|
76
14
|
* Maximum segments to retire in this cycle (default 100). It also bounds the purges attempted (a purge the registry
|
|
77
|
-
* refuses is not charged; {@link RetireExpiredResult.purgeFaults} says how purging stops after refusals)
|
|
78
|
-
* due-index pointers to nothing one call removes
|
|
15
|
+
* refuses is not charged; {@link RetireExpiredResult.purgeFaults} says how purging stops after refusals), the
|
|
16
|
+
* due-index pointers to nothing one call removes, and, counted apart from the retirements, the tombstones this sweep
|
|
17
|
+
* did not write whose pointers one call removes.
|
|
79
18
|
*/
|
|
80
19
|
readonly limit?: number;
|
|
81
20
|
/** Report what would be retired and change nothing. */
|
|
@@ -209,7 +148,10 @@ export interface RetireExpiredResult {
|
|
|
209
148
|
readonly wouldRetire: number;
|
|
210
149
|
/** Tombstone rows actually deleted. Zero under `dryRun`; the `would-purge-tombstone` entries carry the preview. */
|
|
211
150
|
readonly tombstonesPurged: number;
|
|
212
|
-
/**
|
|
151
|
+
/**
|
|
152
|
+
* True when `limit` cut the cycle short — **more segments are still eligible**. Re-run. The cap on removing pointers
|
|
153
|
+
* of tombstones this sweep did not write leaves it false.
|
|
154
|
+
*/
|
|
213
155
|
readonly limited: boolean;
|
|
214
156
|
/**
|
|
215
157
|
* Deletes this sweep attempted that were **refused for a reason other than a lost race**: a tombstone's purge
|
|
@@ -221,7 +163,7 @@ export interface RetireExpiredResult {
|
|
|
221
163
|
*
|
|
222
164
|
* A refused purge is not charged to `limit`, so retirements go on: a tombstone that cannot be purged does not hold
|
|
223
165
|
* the segments behind it past their expiry. **Purging stops for the rest of the call after
|
|
224
|
-
*
|
|
166
|
+
* 3 refused purges in a row**, and a purge that succeeds starts the count
|
|
225
167
|
* again. A blanket refusal therefore costs three attempts a call, and a refusal particular to one row (a blob with a
|
|
226
168
|
* snapshot, an object under a legal hold) holds nothing behind it. The next call tries again. Check this field; a
|
|
227
169
|
* ledger entry for each is `skipped`, but a caller that reads only `retired` sees none of it.
|
|
@@ -238,6 +180,67 @@ export interface RetireExpiredResult {
|
|
|
238
180
|
* or a paused schedule recovers on its own, short enough that a cycle is eight list calls rather than hundreds.
|
|
239
181
|
*/
|
|
240
182
|
export declare const DEFAULT_LOOKBACK_BUCKETS = 7;
|
|
183
|
+
/** The options of a sweep, validated, with every default filled in. */
|
|
184
|
+
interface ResolvedRetireOptions {
|
|
185
|
+
readonly audit: IAuditSink | undefined;
|
|
186
|
+
readonly namespace: string | undefined;
|
|
187
|
+
readonly now: number;
|
|
188
|
+
readonly limit: number;
|
|
189
|
+
readonly purgeTombstones: boolean;
|
|
190
|
+
readonly grace: number;
|
|
191
|
+
readonly maxScanSegments: number;
|
|
192
|
+
readonly dryRun: boolean;
|
|
193
|
+
readonly scan: 'fleet' | 'index';
|
|
194
|
+
readonly lookbackBuckets: number;
|
|
195
|
+
readonly shards: readonly number[] | undefined;
|
|
196
|
+
readonly totalShards: number;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Check a sweep's options and fill in the defaults. Pure: it reads nothing but `options`, and throws
|
|
200
|
+
* `ValidationError` for the first bad one, in the order the checks below run.
|
|
201
|
+
*/
|
|
202
|
+
export declare function resolveRetireOptions(options: RetireExpiredOptions): ResolvedRetireOptions;
|
|
203
|
+
/**
|
|
204
|
+
* The counters and the ledger of one sweep, and the two budgets that bound it.
|
|
205
|
+
*
|
|
206
|
+
* The budget is charged on ATTEMPT, not on success, and that distinction is the whole guard. `dropSegment`
|
|
207
|
+
* writes the tombstone BEFORE sweeping Storage, so a fault in the Storage phase is a segment that is
|
|
208
|
+
* already retired. Counting only successes would let a partial storage outage march through the entire fleet
|
|
209
|
+
* with the cap never engaging, reporting `retired: 0, limited: false` (a "completed sweep that retired nothing")
|
|
210
|
+
* while every segment in the namespace is tombstoned. So a step charges `attempted` before the call that can delete
|
|
211
|
+
* (the drop, or the purge's collection), after any re-read that finds nothing due, and gives the charge back in two
|
|
212
|
+
* places only: a purge the registry or storage refused (`refusedPurge`), and a drop that wrote and deleted nothing
|
|
213
|
+
* because the row was no longer the sweep's to retire (`dropSegmentFor` returned `null`).
|
|
214
|
+
*/
|
|
215
|
+
export declare class SweepTally {
|
|
216
|
+
readonly entries: RetireEntry[];
|
|
217
|
+
eligible: number;
|
|
218
|
+
retired: number;
|
|
219
|
+
wouldRetire: number;
|
|
220
|
+
tombstonesPurged: number;
|
|
221
|
+
limited: boolean;
|
|
222
|
+
/** Deletes the registry refused for a reason other than a lost race. */
|
|
223
|
+
purgeFaults: number;
|
|
224
|
+
/** The first one's reason. A lost race is the row or pointer changing under a fenced delete, which is the fence working. */
|
|
225
|
+
firstPurgeFault: `failed: ${string}` | undefined;
|
|
226
|
+
/** Retirements and tombstone purges begun, charged against `limit`. */
|
|
227
|
+
attempted: number;
|
|
228
|
+
/** Tombstones the sweep did not write whose pointers it worked, bounded by `limit` apart from `attempted`. */
|
|
229
|
+
attemptedForeign: number;
|
|
230
|
+
/**
|
|
231
|
+
* Purging stops after this many refused purges in a row, and a purge that succeeds starts the count again. Each
|
|
232
|
+
* refused attempt costs reads before it fails, and a blanket refusal would fail them all; one row's refusal must
|
|
233
|
+
* not hold the rest.
|
|
234
|
+
*/
|
|
235
|
+
purgeRun: number;
|
|
236
|
+
purging: boolean;
|
|
237
|
+
/** Record a step's ledger entry; `null` adds nothing (a step with nothing to report, or one that recorded its own). */
|
|
238
|
+
add(entry: RetireEntry | null): void;
|
|
239
|
+
/** Count a delete the registry refused, unless it was a lost race. */
|
|
240
|
+
readonly noteFault: (err: unknown) => void;
|
|
241
|
+
/** A purge the registry refused is not charged to `limit`, and after enough in a row purging stops. */
|
|
242
|
+
refusedPurge(): void;
|
|
243
|
+
}
|
|
241
244
|
/**
|
|
242
245
|
* Retire every segment whose retention policy has expired, and clean up the tombstones earlier sweeps left.
|
|
243
246
|
*
|
|
@@ -246,3 +249,4 @@ export declare const DEFAULT_LOOKBACK_BUCKETS = 7;
|
|
|
246
249
|
* condition, not a partial result to be mistaken for a complete sweep.
|
|
247
250
|
*/
|
|
248
251
|
export declare function retireExpired(deps: DropDeps, options: RetireExpiredOptions): Promise<RetireExpiredResult>;
|
|
252
|
+
export {};
|
package/dist/core/retention.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { ChurnDeps } from './leases.js';
|
|
2
2
|
import type { GovernanceMeta, IRegistryDriver, SegmentRef } from './ports.js';
|
|
3
3
|
/**
|
|
4
4
|
* The smallest accepted `expiresAt`: 2001-09-09, the point below which an epoch-**seconds** value would land.
|
|
@@ -59,7 +59,7 @@ export declare function withoutSweepMark(meta: GovernanceMeta | undefined): Gove
|
|
|
59
59
|
* The three-way answer exists so a fleet sweep can *report* a malformed policy instead of choosing between two
|
|
60
60
|
* bad options: treating it as "no policy" hides an operator error on a segment someone believes is expiring, and
|
|
61
61
|
* throwing aborts a ledger over one bad row. Nothing writes an invalid policy through
|
|
62
|
-
*
|
|
62
|
+
* `store.setRetention` — hand-edited rows, an older/newer writer, or a restore are how it happens.
|
|
63
63
|
*/
|
|
64
64
|
export declare function readRetentionPolicy(meta: GovernanceMeta | undefined): RetentionPolicy | null | 'invalid';
|
|
65
65
|
/** The stored policy for one segment, or `null` if it has none (and `'invalid'` for a malformed one). */
|
package/dist/core/retry.d.ts
CHANGED
|
@@ -1,22 +1,8 @@
|
|
|
1
|
+
import type { Clock, Rng } from './determinism.js';
|
|
1
2
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
* A pure, storage-agnostic helper: it knows nothing about HTTP, AWS, or any SDK. It retries an operation
|
|
5
|
-
* while the operation keeps failing with a **retryable** error (by default {@link TransientError}), waiting
|
|
6
|
-
* a bounded, jittered, exponentially-growing delay between attempts. Time and randomness are **injected**
|
|
7
|
-
* (the determinism seam) — production wires a `setTimeout`-backed clock + a real RNG; the
|
|
8
|
-
* tests wire virtual ones — so a backoff schedule is replayable from a seed and unit tests run with no
|
|
9
|
-
* real sleeping.
|
|
10
|
-
*
|
|
11
|
-
* It deliberately does **not** retry deterministic failures ({@link WriteConflictError},
|
|
12
|
-
* {@link ValidationError}, {@link IntegrityError}, {@link NotFoundError}, …): retrying those either can't
|
|
13
|
-
* help or would be incorrect. A pointer conflict is retried by a *separate* loop inside
|
|
14
|
-
* the publish, because each attempt must re-read the row and re-decide — a blind replay of the same
|
|
15
|
-
* compare-and-swap would either fail again on a stale token or, worse, advance a pointer whose state has
|
|
16
|
-
* changed underneath it. A write-once object collision is never replayed at all: the generation number is
|
|
17
|
-
* taken, so the caller has to pick a new one.
|
|
3
|
+
* How the store retries a transient read: attempts, exponential backoff with a cap, and jitter. By default 4 attempts,
|
|
4
|
+
* a 50 ms base delay, ×2, a 2 s cap and full jitter; the store's `retry` option overrides them a field at a time.
|
|
18
5
|
*/
|
|
19
|
-
import type { Clock, Rng } from './determinism.js';
|
|
20
6
|
export interface RetryPolicy {
|
|
21
7
|
/** Total attempts including the first (so `maxAttempts: 4` = 1 try + 3 retries). Must be ≥ 1. */
|
|
22
8
|
readonly maxAttempts: number;
|
|
@@ -61,6 +47,15 @@ export interface RetryDeps {
|
|
|
61
47
|
export declare function backoffDelayMs(policy: RetryPolicy, attempt: number): number;
|
|
62
48
|
/** Apply the policy's jitter to a computed delay using the injected RNG. */
|
|
63
49
|
export declare function applyJitter(policy: RetryPolicy, delayMs: number, rng: Rng): number;
|
|
50
|
+
/** A clock that can wait: the one member {@link jitteredWait} uses. */
|
|
51
|
+
interface Sleeper {
|
|
52
|
+
readonly sleep?: ((ms: number) => Promise<void>) | undefined;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Wait a uniformly spread share of `boundMs`, whole milliseconds, by the clock's `sleep`. With no `rng` it waits the
|
|
56
|
+
* whole bound; with no `sleep` it does not wait.
|
|
57
|
+
*/
|
|
58
|
+
export declare function jitteredWait(clock: Sleeper | undefined, rng: Rng | undefined, boundMs: number): Promise<void>;
|
|
64
59
|
/**
|
|
65
60
|
* The store's read retry, as a write path takes it for the reads it makes along the way: a policy (the default when
|
|
66
61
|
* absent) and what {@link withRetry} needs to run it.
|
|
@@ -78,7 +73,8 @@ export declare function retryRead<T>(op: () => Promise<T>, retry: ReadRetry | un
|
|
|
78
73
|
/**
|
|
79
74
|
* `policy`, checked when it is wired rather than at the first retry: a delay or factor that is not a finite number, a
|
|
80
75
|
* negative delay, a factor below 1 or a jitter mode it does not know would make the backoff `NaN`, which a timer reads
|
|
81
|
-
* as 1 ms, so bounded backoff silently became a hot retry loop.
|
|
76
|
+
* as 1 ms, so bounded backoff silently became a hot retry loop. `maxAttempts` is a whole number, and `maxDelayMs` is not
|
|
77
|
+
* below `baseDelayMs`, which it would otherwise cap every delay under.
|
|
82
78
|
*/
|
|
83
79
|
export declare function checkedRetryPolicy(policy: RetryPolicy): RetryPolicy;
|
|
84
80
|
/**
|
|
@@ -87,3 +83,4 @@ export declare function checkedRetryPolicy(policy: RetryPolicy): RetryPolicy;
|
|
|
87
83
|
* the *operation's* error — never a wrapper — so callers keep their typed-error branching.
|
|
88
84
|
*/
|
|
89
85
|
export declare function withRetry<T>(op: () => Promise<T>, policy: RetryPolicy, deps: RetryDeps): Promise<T>;
|
|
86
|
+
export {};
|
package/dist/core/rollback.d.ts
CHANGED
|
@@ -1,21 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `listGenerations` and `rollbackSegment` — see what a segment has been, and put it back.
|
|
3
|
-
*
|
|
4
|
-
* Immutable generations mean the previous version of a segment is usually still sitting in the bucket: a load
|
|
5
|
-
* that replaced it did not overwrite anything, it wrote a new object and moved a pointer. So recovering from a
|
|
6
|
-
* bad load is, in principle, moving the pointer back, which no load, publish or sweep can do, because every other
|
|
7
|
-
* write path in the library only ever advances the pointer, and refuses a move that would lower it.
|
|
8
|
-
*
|
|
9
|
-
* That refusal is right for a *writer*: a load whose ids came from upstream loses nothing by being out-raced, and
|
|
10
|
-
* letting it regress the pointer would let a slow loader silently undo a fast one. It is wrong for an *operator*,
|
|
11
|
-
* who has looked at the segment, decided the current generation is wrong, and knows which one they want. So
|
|
12
|
-
* rollback is the one call that moves the pointer backwards, and it is deliberately not reachable from any
|
|
13
|
-
* automatic path — no sweep, no retry, no reconciliation calls it.
|
|
14
|
-
*
|
|
15
|
-
* It is audited for the same reason. Every other pointer move can be reconstructed from "a load happened"; this
|
|
16
|
-
* one is a human overriding the ordering rule the rest of the system relies on, which is exactly the event an
|
|
17
|
-
* Art. 30 record or an incident review wants to find.
|
|
18
|
-
*/
|
|
19
1
|
import { type IAuditSink } from './audit.js';
|
|
20
2
|
import type { IKeystore } from './crypto.js';
|
|
21
3
|
import { type ChurnDeps } from './leases.js';
|
|
@@ -26,8 +8,10 @@ export interface GenerationListDeps extends ChurnDeps {
|
|
|
26
8
|
readonly registry: IRegistryDriver;
|
|
27
9
|
/**
|
|
28
10
|
* The key of an encrypted segment, so a rollback can read its target's count and metadata and seal them into the
|
|
29
|
-
* row. Without it, a rollback of an encrypted segment
|
|
30
|
-
* generation it moves to, which a reader then takes from the object
|
|
11
|
+
* row. Without it, a rollback of an encrypted segment onto a generation the row knows still works and leaves the
|
|
12
|
+
* row with no summary of the generation it moves to, which a reader then takes from the object; `allowUnknown` needs
|
|
13
|
+
* it, and is refused with `KeyUnavailableError`, or throws `TransientError` when the key service did not answer in
|
|
14
|
+
* time (the keystore's own, or one whose `cause` is the timeout it threw).
|
|
31
15
|
*/
|
|
32
16
|
readonly keystore?: IKeystore;
|
|
33
17
|
}
|
|
@@ -36,6 +20,15 @@ export interface GenerationEntry {
|
|
|
36
20
|
readonly generation: number;
|
|
37
21
|
/** Whether the registry pointer currently names this generation. */
|
|
38
22
|
readonly current: boolean;
|
|
23
|
+
/**
|
|
24
|
+
* Whether the row's record of the generations it published, `knownGens`, names this generation. A rollback onto a known
|
|
25
|
+
* generation needs no `allowUnknown`; an unknown one needs it, except below the pointer in the two cases the pointer
|
|
26
|
+
* alone decides (a row with no usable `knownGens`, and a target below a full one's bound), where it needs none. The
|
|
27
|
+
* row's word, read with the listing's one registry read: the object is not opened, so
|
|
28
|
+
* its checksum is not compared. `false` for every generation of a row whose record is missing or unusable, and for a
|
|
29
|
+
* segment with no row. The current generation of a row with a usable record is always known.
|
|
30
|
+
*/
|
|
31
|
+
readonly known: boolean;
|
|
39
32
|
/**
|
|
40
33
|
* The current generation's id count, from its registry row, with no read of the object. Present only on the current
|
|
41
34
|
* entry, and only when the row's summary can be used: for an encrypted segment that takes a keystore that opens the
|
|
@@ -58,6 +51,7 @@ export interface GenerationEntry {
|
|
|
58
51
|
* carries the id count and metadata its row records, from the same read.
|
|
59
52
|
*/
|
|
60
53
|
export declare function listGenerations(ref: SegmentRef, deps: GenerationListDeps): Promise<GenerationEntry[]>;
|
|
54
|
+
/** What a rollback moved: the generation the pointer named before, and the one it names now. */
|
|
61
55
|
export interface RollbackResult {
|
|
62
56
|
/** The generation the pointer named before the rollback. `null` when the segment had no current generation. */
|
|
63
57
|
readonly fromGeneration: number | null;
|
|
@@ -65,11 +59,20 @@ export interface RollbackResult {
|
|
|
65
59
|
readonly generation: number;
|
|
66
60
|
}
|
|
67
61
|
/**
|
|
68
|
-
* Move a segment's pointer
|
|
62
|
+
* Move a segment's pointer to a generation that is still in the bucket and that the segment's row knows it published.
|
|
69
63
|
*
|
|
70
64
|
* The only non-forward-only publish in the library, and the only one an operator has to ask for by name. It
|
|
71
65
|
* refuses rather than guesses:
|
|
72
66
|
*
|
|
67
|
+
* - a generation the **row does not know** is refused with `ValidationError`, before any request to storage. The row
|
|
68
|
+
* records, in `knownGens`, the generations its loads and rollbacks made current, with the footer checksum of each
|
|
69
|
+
* object; a target outside that record is where objects live that no row ever published: a load that wrote its object
|
|
70
|
+
* and died before the publish, a refused load's object, an object sealed under a key no row holds, a load published
|
|
71
|
+
* after the point a registry was restored to. {@link rollbackSegment}'s `allowUnknown` is the one explicit override;
|
|
72
|
+
* - a known generation whose object is **not the one the row recorded** (its footer checksum differs: another object was
|
|
73
|
+
* stored under the number) is refused with `IntegrityError`, with or without a key, since the footer is in the clear.
|
|
74
|
+
* The check runs before the target is opened, so an object sealed under another key reports as another object, not as a
|
|
75
|
+
* failed decryption. `allowUnknown` moves onto it anyway;
|
|
73
76
|
* - a generation whose object is **not in the bucket** — collected, or never written — is refused with
|
|
74
77
|
* `NotFoundError` rather than pointed at, because a pointer naming a missing object is the one state the whole
|
|
75
78
|
* design exists to avoid;
|
|
@@ -78,23 +81,50 @@ export interface RollbackResult {
|
|
|
78
81
|
* - a target that is **not what the row says the segment is** is refused with `IntegrityError`: a cleartext object
|
|
79
82
|
* under a row with keys, or an encrypted one under a row with none, either of which every read would then refuse.
|
|
80
83
|
* One tail read of the target decides it, as does a footer that fails its own checks;
|
|
81
|
-
* -
|
|
84
|
+
* - `allowUnknown` on an **encrypted** segment, outside the two cases the pointer alone decides (below), needs the key,
|
|
85
|
+
* and is refused with `KeyUnavailableError` when the store has
|
|
86
|
+
* no keystore, or its keystore holds none of the key's KEKs or one that does not unwrap the key: an object the row
|
|
87
|
+
* does not vouch for and that nothing opens may be one sealed under a key no row holds. A key service that did not
|
|
88
|
+
* answer in time is thrown as `TransientError`, to retry;
|
|
89
|
+
* - rolling to the generation **already current** is a no-op that reports itself, not an error, once one footer read finds
|
|
90
|
+
* its object in the bucket and, when the row knows it, with the checksum recorded. A pointer that names a missing
|
|
91
|
+
* object is refused with `NotFoundError`, as any missing target is, and one that names another object than the row
|
|
92
|
+
* recorded with `IntegrityError`.
|
|
93
|
+
*
|
|
94
|
+
* Which generations are known: the current one, the ones a load's `keep` keeps below it, those a live lease holds, and
|
|
95
|
+
* the ones a rollback left above the pointer, until a load numbers above them. A known target needs no flag and, for
|
|
96
|
+
* the same reason, no key: moving onto an object the row published, forward or back. Two cases are decided by the
|
|
97
|
+
* pointer alone, so a target below it needs no flag and no key, and are audited with `unknown: true`: a row with no
|
|
98
|
+
* usable `knownGens` (a registry that drops the field, or a row whose pointer was moved by hand); and a target below the
|
|
99
|
+
* bound of a full record (128 entries or more, leased ones included), which is its least recently current entry that no
|
|
100
|
+
* lease entry names. Neither gets the checks above. A generation a publish with a smaller `keep` no longer names but that
|
|
101
|
+
* stays in the bucket (a lease spared it, or its delete failed) needs `allowUnknown` until a later listing collects it.
|
|
82
102
|
*
|
|
83
103
|
* The same tail read opens the target when the segment's key is at hand, and the rollback writes the target's own id
|
|
84
104
|
* count and metadata into the row, in the compare-and-swap that moves the pointer: a reader that sees the target as
|
|
85
105
|
* current sees what describes it. (One tail read, and a range read as well when the target's index is longer than the
|
|
86
|
-
* tail read: a target of very many chunks.) A store with no keystore, or
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
* the
|
|
90
|
-
* the
|
|
91
|
-
*
|
|
106
|
+
* tail read: a target of very many chunks.) A store with no keystore, or whose keystore cannot open the segment's key
|
|
107
|
+
* (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out as
|
|
108
|
+
* `AbortSignal.timeout()` and the AWS SDK report a timeout), still rolls an encrypted segment onto a known generation, or
|
|
109
|
+
* below the pointer in the two judged-as-before cases, and leaves the row with no summary of the target. Any other
|
|
110
|
+
* failure of the keystore is thrown. After the swap the target's footer is read once more, and the pointer put back,
|
|
111
|
+
* restoring the summary and the record the row had, unless it is still the object that was read: collection can delete a
|
|
112
|
+
* target below the old pointer between the read and the swap, and an erasure can delete one above it and a load take its
|
|
113
|
+
* number. A swap that gets no answer is settled by reading the row: one the row shows landed, the row being the one the
|
|
114
|
+
* swap leaves, is checked as any swap is, and otherwise its `TransientError` is thrown, since it may have landed or may
|
|
115
|
+
* still land, and a re-run finds the pointer where it is. A write by another writer is not taken for it, unless it leaves
|
|
116
|
+
* the very row this swap would. On a cleartext segment two writes can: another rollback onto the same target, whatever
|
|
117
|
+
* row it read, and a load that wrote under that number the very object the rollback read and carried the ledger the
|
|
118
|
+
* rollback would (a load that carries a usable ledger, as one with a `keep` above 64 does; over a row with no usable
|
|
119
|
+
* ledger the rollback writes none and a load seeds one, so those rows differ). On an encrypted segment a rollback with no
|
|
120
|
+
* key at hand writes no summary, so another such rollback onto the same target can too. That call then reports and
|
|
121
|
+
* audits the move as its own; the pointer is where it asked for it.
|
|
92
122
|
*
|
|
93
123
|
* What it does **not** do is delete anything. The generations above the new pointer stay in the bucket, which is
|
|
94
124
|
* what makes the rollback reversible — roll forward again by naming one of them. They are also then *above*
|
|
95
125
|
* `currentGen`, where generation collection never looks. They stay until loads pass them (the first load whose
|
|
96
|
-
* number one of them holds numbers above them all, and collection then keeps
|
|
97
|
-
*
|
|
126
|
+
* number one of them holds numbers above them all, and collection then keeps `keep` of what is below its pointer, the
|
|
127
|
+
* generation that load superseded first), `dropSegment` deletes them, or a subject erasure
|
|
98
128
|
* deletes them: all of those present when it rewrites, and, when the current generation does not hold the id, only those
|
|
99
129
|
* holding it and any no read of the segment can open, whatever the id.
|
|
100
130
|
* Not collecting them is a deliberate trade: an operator who has just undone a bad load should not have the
|
|
@@ -103,5 +133,5 @@ export interface RollbackResult {
|
|
|
103
133
|
*/
|
|
104
134
|
export declare function rollbackSegment(ref: SegmentRef, toGeneration: number, deps: GenerationListDeps, options?: {
|
|
105
135
|
audit?: IAuditSink;
|
|
106
|
-
|
|
136
|
+
allowUnknown?: boolean;
|
|
107
137
|
}): Promise<RollbackResult>;
|
package/dist/core/validate.d.ts
CHANGED
|
@@ -49,3 +49,5 @@ export declare function validateUserRef(ref: SegmentRef): void;
|
|
|
49
49
|
export declare function validateChunkKeyOrder(keys: readonly number[]): void;
|
|
50
50
|
/** Validate a chunk ref: the segment/namespace rules plus `chunkKey` ∈ `[0, 65535]` (a u16). A driver-boundary check. */
|
|
51
51
|
export declare function validateChunkRef(ref: ChunkRef): void;
|
|
52
|
+
/** The chunk-key half of {@link validateChunkRef}, for a caller that has already validated the segment: an integer in `[0, 65535]`. */
|
|
53
|
+
export declare function validateChunkKey(chunkKey: number): void;
|
package/dist/driver-kit.d.ts
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
export type { GenKey, IRegistryDriver, IStorageDriver, SegmentRef, StorageBackend, StorageCaps, StorageDeleteOptions, TailRead, } from './core/ports.js';
|
|
2
2
|
export { brandAsBackend, STORAGE_BACKEND } from './core/ports.js';
|
|
3
|
-
export type {
|
|
4
|
-
export {
|
|
3
|
+
export type { ClearRegistrySummary, GenerationMetadata, GovernanceMeta, KnownGeneration, LeaseEntry, NewRegistryRecord, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, RegistryWriteOptions, SealedRegistrySummary, Token, } from './core/ports.js';
|
|
4
|
+
export type { ChunkRead, ChunkRef, GenerationSummary, ReadChunksOptions, SegmentSize, StorageChunkSource, } from './core/ports.js';
|
|
5
|
+
export type { Aead, AeadSealed, IKeystore, WrappedDek } from './core/crypto.js';
|
|
6
|
+
export type { Entropy } from './core/determinism.js';
|
|
7
|
+
export type { IMetricsSink, MetricEvent, MetricOpName } from './core/metrics.js';
|
|
8
|
+
export { CloudRoaringError, errorKind, IntegrityError, NotFoundError, TransientError, ValidationError, WriteConflictError, } from './core/errors.js';
|
|
9
|
+
export type { ErrorKind } from './core/errors.js';
|
|
5
10
|
export { RESOLVED_FIELDS, pointerIdOf, renewPointer, renewsPointer } from './core/pointer-id.js';
|
|
6
11
|
export { validateSegmentRef } from './core/validate.js';
|
|
7
12
|
export type { BlobSink } from './core/blob.js';
|
|
@@ -9,3 +14,7 @@ export { MAX_ROW_BYTES, ObjectStoreRegistry, ObjectVersionRaced, } from './drive
|
|
|
9
14
|
export type { ObjectRegistryStore, ObjectRow } from './drivers/_shared/object-registry.js';
|
|
10
15
|
export { encodeNameForKey, encodeNameForPath, namespaceKeyPart, namespacePathPart, } from './drivers/_shared/keys.js';
|
|
11
16
|
export { normalizeObjectPrefix, prefixPart } from './drivers/_shared/object-registry-keys.js';
|
|
17
|
+
export { parseGenerationFromKey, segmentObjectPrefix, storageObjectKey, } from './drivers/_shared/object-keys.js';
|
|
18
|
+
export { refuseUnknownOptions, requireBool, requirePositiveInt } from './drivers/_shared/options.js';
|
|
19
|
+
export { isTransportFault, toTransient } from './drivers/_shared/faults.js';
|
|
20
|
+
export { concatBytes } from './drivers/_shared/bytes.js';
|