@cloudbitmaps/core 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +4 -4
- package/README.md +33 -16
- package/dist/chunk-5OMC5U7U.js +508 -0
- package/dist/chunk-5OMC5U7U.js.map +7 -0
- package/dist/core/audit.d.ts +128 -0
- package/dist/core/bit-route.d.ts +21 -0
- package/dist/core/blob.d.ts +34 -0
- package/dist/core/budget.d.ts +68 -0
- package/dist/core/codec.d.ts +122 -0
- package/dist/core/concurrency.d.ts +15 -0
- package/dist/core/consistency.d.ts +73 -0
- package/dist/core/cooperative.d.ts +78 -0
- package/dist/core/cost.d.ts +254 -0
- package/dist/core/crbm/crc32c.d.ts +10 -0
- package/dist/core/crbm/format.d.ts +114 -0
- package/dist/core/crbm/reader.d.ts +86 -0
- package/dist/core/crbm/varint.d.ts +13 -0
- package/dist/core/crbm/writer.d.ts +39 -0
- package/dist/core/crbm-storage-source.d.ts +522 -0
- package/dist/core/crypto.d.ts +80 -0
- package/dist/core/determinism.d.ts +42 -0
- package/dist/core/discover.d.ts +77 -0
- package/dist/core/due-index.d.ts +53 -0
- package/dist/core/engine.d.ts +246 -0
- package/dist/core/erase-id.d.ts +176 -0
- package/dist/core/erasure.d.ts +185 -0
- package/dist/core/errors.d.ts +113 -0
- package/dist/core/generation-gc.d.ts +56 -0
- package/dist/core/keys.d.ts +23 -0
- package/dist/core/load.d.ts +121 -0
- package/dist/core/lru.d.ts +69 -0
- package/dist/core/metrics.d.ts +136 -0
- package/dist/core/name-codec.d.ts +69 -0
- package/dist/core/pinned-storage-source.d.ts +90 -0
- package/dist/core/ports.d.ts +423 -0
- package/dist/core/reader-defaults.d.ts +18 -0
- package/dist/core/registry-scan.d.ts +47 -0
- package/dist/core/retention-sweep.d.ts +198 -0
- package/dist/core/retention.d.ts +67 -0
- package/dist/core/retry.d.ts +69 -0
- package/dist/core/rollback.d.ts +75 -0
- package/dist/core/validate.d.ts +44 -0
- package/dist/driver-kit.d.ts +9 -0
- package/dist/driver-kit.js +275 -0
- package/dist/driver-kit.js.map +7 -0
- package/dist/drivers/_shared/keys.d.ts +26 -0
- package/dist/drivers/_shared/object-registry-keys.d.ts +28 -0
- package/dist/drivers/_shared/object-registry.d.ts +89 -0
- package/dist/drivers/_shared/registry.d.ts +66 -0
- package/dist/drivers/backends.d.ts +43 -0
- package/dist/drivers/crypto.d.ts +38 -0
- package/dist/drivers/localfs/fs-util.d.ts +21 -0
- package/dist/drivers/localfs/paths.d.ts +33 -0
- package/dist/drivers/localfs/registry.d.ts +28 -0
- package/dist/drivers/localfs/storage.d.ts +19 -0
- package/dist/drivers/memory.d.ts +49 -0
- package/dist/drivers/retry/retrying-chunk-source.d.ts +59 -0
- package/dist/export/index.d.ts +93 -0
- package/dist/index.d.ts +76 -2760
- package/dist/index.js +3527 -3133
- package/dist/index.js.map +7 -1
- package/dist/testing/fuzz-core.d.ts +14 -0
- package/package.json +19 -115
- package/dist/azure/index.cjs +0 -368
- package/dist/azure/index.cjs.map +0 -1
- package/dist/azure/index.d.cts +0 -48
- package/dist/azure/index.d.ts +0 -48
- package/dist/azure/index.js +0 -304
- package/dist/azure/index.js.map +0 -1
- package/dist/cassandra/index.cjs +0 -265
- package/dist/cassandra/index.cjs.map +0 -1
- package/dist/cassandra/index.d.cts +0 -52
- package/dist/cassandra/index.d.ts +0 -52
- package/dist/cassandra/index.js +0 -204
- package/dist/cassandra/index.js.map +0 -1
- package/dist/chunk-7LMLYSVJ.js +0 -43
- package/dist/chunk-7LMLYSVJ.js.map +0 -1
- package/dist/chunk-BD22WDAN.js +0 -230
- package/dist/chunk-BD22WDAN.js.map +0 -1
- package/dist/chunk-NUIDEEFZ.js +0 -91
- package/dist/chunk-NUIDEEFZ.js.map +0 -1
- package/dist/chunk-OZ6LAFV4.js +0 -6
- package/dist/chunk-OZ6LAFV4.js.map +0 -1
- package/dist/chunk-SNJVZ227.js +0 -35
- package/dist/chunk-SNJVZ227.js.map +0 -1
- package/dist/dynamodb/index.cjs +0 -767
- package/dist/dynamodb/index.cjs.map +0 -1
- package/dist/dynamodb/index.d.cts +0 -106
- package/dist/dynamodb/index.d.ts +0 -106
- package/dist/dynamodb/index.js +0 -469
- package/dist/dynamodb/index.js.map +0 -1
- package/dist/gcs/index.cjs +0 -343
- package/dist/gcs/index.cjs.map +0 -1
- package/dist/gcs/index.d.cts +0 -46
- package/dist/gcs/index.d.ts +0 -46
- package/dist/gcs/index.js +0 -279
- package/dist/gcs/index.js.map +0 -1
- package/dist/index.cjs +0 -4990
- package/dist/index.cjs.map +0 -1
- package/dist/index.d.cts +0 -2761
- package/dist/mongodb/index.cjs +0 -260
- package/dist/mongodb/index.cjs.map +0 -1
- package/dist/mongodb/index.d.cts +0 -45
- package/dist/mongodb/index.d.ts +0 -45
- package/dist/mongodb/index.js +0 -199
- package/dist/mongodb/index.js.map +0 -1
- package/dist/mysql/index.cjs +0 -281
- package/dist/mysql/index.cjs.map +0 -1
- package/dist/mysql/index.d.cts +0 -56
- package/dist/mysql/index.d.ts +0 -56
- package/dist/mysql/index.js +0 -211
- package/dist/mysql/index.js.map +0 -1
- package/dist/ports-CvcvJWf6.d.cts +0 -395
- package/dist/ports-CvcvJWf6.d.ts +0 -395
- package/dist/postgres/index.cjs +0 -273
- package/dist/postgres/index.cjs.map +0 -1
- package/dist/postgres/index.d.cts +0 -54
- package/dist/postgres/index.d.ts +0 -54
- package/dist/postgres/index.js +0 -203
- package/dist/postgres/index.js.map +0 -1
- package/dist/redis/index.cjs +0 -257
- package/dist/redis/index.cjs.map +0 -1
- package/dist/redis/index.d.cts +0 -42
- package/dist/redis/index.d.ts +0 -42
- package/dist/redis/index.js +0 -197
- package/dist/redis/index.js.map +0 -1
- package/dist/s3/index.cjs +0 -918
- package/dist/s3/index.cjs.map +0 -1
- package/dist/s3/index.d.cts +0 -112
- package/dist/s3/index.d.ts +0 -112
- package/dist/s3/index.js +0 -555
- package/dist/s3/index.js.map +0 -1
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The retention **sweep** — the thing that acts on the policies `setSegmentRetention` records.
|
|
3
|
+
*
|
|
4
|
+
* `retireExpired` enumerates the registry, selects the segments whose `expiresAt` has passed, and retires each
|
|
5
|
+
* one through {@link dropSegment}. It deliberately **delegates rather than reimplements**: the registry → Storage
|
|
6
|
+
* ordering, the re-sweep for an object a load was still writing, and the `generationsRemaining` report are all
|
|
7
|
+
* load-bearing and already live there. A sweep that open-coded the deletions would be a second
|
|
8
|
+
* implementation of the most dangerous ordering in the library.
|
|
9
|
+
*
|
|
10
|
+
* **This is a call, not a daemon.** Nothing here schedules itself. You run it from whatever heartbeat your
|
|
11
|
+
* deployment already has — an EventBridge rule, a Kubernetes CronJob, a queue consumer, the job that runs your
|
|
12
|
+
* loads — and the library stays a library: it has to behave identically in a Lambda, an edge isolate and a
|
|
13
|
+
* long-lived server, and a timer that only works in one of those is worse than none.
|
|
14
|
+
*
|
|
15
|
+
* Three properties make it safe to point at a fleet:
|
|
16
|
+
*
|
|
17
|
+
* - **`dryRun` at the sweep level.** `dropSegment`'s `confirmSegment` guard is vacuous in a loop (it is the same
|
|
18
|
+
* variable twice), so the meaningful preview is here: it reports what it *would* retire, per segment, and
|
|
19
|
+
* touches nothing.
|
|
20
|
+
* - **A per-cycle `limit`.** Clock skew, a bad backfill, or a namespace-wide policy mistake should cost one
|
|
21
|
+
* bounded batch, not the fleet. When the limit bites, the result says so (`limited: true`) rather than looking
|
|
22
|
+
* like a completed sweep.
|
|
23
|
+
* - **A ledger, never an exception.** A per-segment fault is recorded and the sweep continues — the caller's
|
|
24
|
+
* load-bearing question is *which* segments were retired, and a throw from the middle of the loop answers it
|
|
25
|
+
* for none of them while having already retired some. Same posture as `eraseNamespace`.
|
|
26
|
+
*
|
|
27
|
+
* It also **purges the tombstones its own retirements leave behind**. A retired segment that had a registry row
|
|
28
|
+
* gets a `destroyed` row, and one dead row per retired daily bucket — or per retired dedup wave — is exactly the
|
|
29
|
+
* registry litter `dropSegment` already refuses to create for a row-less accumulator. Purging is narrow on
|
|
30
|
+
* purpose: only a tombstone carrying the **stamp this sweep writes on its own retirements** (so it is attributably
|
|
31
|
+
* ours: a crypto-shred's tombstone does not carry it, unless the shred lands between this sweep's re-read of the
|
|
32
|
+
* row and its drop, which then reports the tombstone dropped and stamps it), only after a grace period, and only
|
|
33
|
+
* once Storage is provably empty for it —
|
|
34
|
+
* because deleting the row is what makes the name reusable and takes the segment out of reach of
|
|
35
|
+
* the generation collection.
|
|
36
|
+
*/
|
|
37
|
+
import { type IAuditSink } from './audit.js';
|
|
38
|
+
import type { DropDeps, DropResult } from './erasure.js';
|
|
39
|
+
/** Default cap on retirements per sweep — a bounded batch, so a policy mistake costs one batch, not the fleet. */
|
|
40
|
+
export declare const DEFAULT_RETIRE_LIMIT = 100;
|
|
41
|
+
/**
|
|
42
|
+
* Default delay before a retirement's own tombstone row is purged: 24 h.
|
|
43
|
+
*
|
|
44
|
+
* The row is a fence — while it exists, a load refuses the segment, so
|
|
45
|
+
* a load that was mid-write when the drop landed cannot resurrect it. That window is
|
|
46
|
+
* seconds to minutes in practice; a day of margin costs one tiny row and removes any need to reason about it.
|
|
47
|
+
*/
|
|
48
|
+
export declare const DEFAULT_TOMBSTONE_GRACE_MS = 86400000;
|
|
49
|
+
export interface RetireExpiredOptions {
|
|
50
|
+
/** Scope the sweep to one namespace. Omit to sweep every namespace the registry knows. */
|
|
51
|
+
readonly namespace?: string;
|
|
52
|
+
/**
|
|
53
|
+
* The instant to compare policies against (epoch-ms). Required here because `core/` takes its time from the
|
|
54
|
+
* caller, never from the platform — `store.retireExpired()` fills it in from the store's clock.
|
|
55
|
+
*/
|
|
56
|
+
readonly now: number;
|
|
57
|
+
/** Maximum segments to retire in this cycle (default 100). */
|
|
58
|
+
readonly limit?: number;
|
|
59
|
+
/** Report what would be retired and change nothing. */
|
|
60
|
+
readonly dryRun?: boolean;
|
|
61
|
+
/** Forwarded to each `dropSegment`, so every retirement lands in the audit trail as `segment.dispose`. */
|
|
62
|
+
readonly audit?: IAuditSink;
|
|
63
|
+
/** Ceiling on rows enumerated (default 250,000); exceeding it throws. */
|
|
64
|
+
readonly maxScanSegments?: number;
|
|
65
|
+
/**
|
|
66
|
+
* **Where the candidates come from.**
|
|
67
|
+
*
|
|
68
|
+
* - `'fleet'` (default) — drain `registry.list()` and filter. Cost tracks the **fleet**, every cycle, even
|
|
69
|
+
* when nothing expires. Complete by construction: it cannot miss a policy.
|
|
70
|
+
* - `'index'` — read only the due buckets of the due index. Cost tracks **what is
|
|
71
|
+
* expiring**. Each candidate's live row is still re-read before anything is decided, so a stale pointer
|
|
72
|
+
* costs one read and retires nothing.
|
|
73
|
+
*
|
|
74
|
+
* **`'index'` is not a drop-in replacement for `'fleet'`; it is the fast half of a pair.** A policy whose
|
|
75
|
+
* pointer write failed (`indexed: false`), or whose ref is too long to index, has no pointer — so a
|
|
76
|
+
* deployment that *only* ever runs `'index'` will never retire those. Run `'fleet'` periodically as the
|
|
77
|
+
* repair pass. The default is `'fleet'` because it is complete by construction.
|
|
78
|
+
*/
|
|
79
|
+
readonly scan?: 'fleet' | 'index';
|
|
80
|
+
/**
|
|
81
|
+
* **The shards this worker owns**, with {@link totalShards}. Without them every replica sweeps the whole
|
|
82
|
+
* fleet and contends over the same segments. A stable hash of the segment key, so a worker owns the same slice
|
|
83
|
+
* across restarts.
|
|
84
|
+
*/
|
|
85
|
+
readonly shards?: readonly number[];
|
|
86
|
+
/** Total shards the fleet is split into. Required with {@link shards}; ignored without it. */
|
|
87
|
+
readonly totalShards?: number;
|
|
88
|
+
/**
|
|
89
|
+
* How many **past** buckets an `'index'` scan reads besides the current one (default 7, one per day).
|
|
90
|
+
* A sweep that did not run — scaled to zero, a failed deploy, a paused
|
|
91
|
+
* schedule — leaves its buckets behind, and this is how far back a later cycle reaches for them. Bounded so a
|
|
92
|
+
* long outage costs a bounded number of list calls per cycle rather than one per day since the epoch;
|
|
93
|
+
* anything older is the `'fleet'` repair pass's job.
|
|
94
|
+
*/
|
|
95
|
+
readonly lookbackBuckets?: number;
|
|
96
|
+
/**
|
|
97
|
+
* Whether to delete the tombstone rows this sweep's own past retirements left (default `true`). Set `false` to
|
|
98
|
+
* keep every one of them — the right choice if something outside this library treats the presence of a
|
|
99
|
+
* `destroyed` row as an attestation. That includes the row of a retirement whose drop found no Storage
|
|
100
|
+
* generation to delete and left none behind: with the default that row is deleted in the same pass, since it
|
|
101
|
+
* would do nothing but fence the name against every writer, and with `false` it stays, stamped like any other
|
|
102
|
+
* of this sweep's tombstones, so a later sweep with purging on deletes it once `tombstoneGraceMs` has passed.
|
|
103
|
+
* While it stays, it fences the name like every kept tombstone. What records a retirement either way is its
|
|
104
|
+
* ledger entry, and its `segment.dispose` event (`generationsDeleted: 0` for an empty one) when an `audit` sink
|
|
105
|
+
* is passed.
|
|
106
|
+
*
|
|
107
|
+
* Two knobs rather than one `number | 'never'`, deliberately: `0` would have had to mean "purge immediately"
|
|
108
|
+
* here while `cache.genTtlMs: 0` in this same library means "never refresh on a timer", and one option whose
|
|
109
|
+
* zero is the opposite of another's is a reading hazard for whoever tunes both.
|
|
110
|
+
*/
|
|
111
|
+
readonly purgeTombstones?: boolean;
|
|
112
|
+
/** How long a retirement's tombstone must age before this sweep deletes it (default 24 h). */
|
|
113
|
+
readonly tombstoneGraceMs?: number;
|
|
114
|
+
}
|
|
115
|
+
/** What happened to one segment in a sweep. */
|
|
116
|
+
export type RetireEntry = {
|
|
117
|
+
readonly segment: string;
|
|
118
|
+
readonly namespace?: string;
|
|
119
|
+
/** Retired: `result` is `dropSegment`'s full report — **check `generationsRemaining`**. */
|
|
120
|
+
readonly action: 'retired';
|
|
121
|
+
readonly expiresAt: number;
|
|
122
|
+
readonly result: DropResult;
|
|
123
|
+
/**
|
|
124
|
+
* Present when the retirement **completed the destructive part and then faulted** — the tombstone is written
|
|
125
|
+
* and the segment reads empty, but something after that (the Storage sweep) threw. The storage may not be fully
|
|
126
|
+
* reclaimed; re-run. Reported as `retired` rather than `skipped` because the segment really is retired, and
|
|
127
|
+
* saying otherwise is the one thing a caller cannot recover from.
|
|
128
|
+
*/
|
|
129
|
+
readonly fault?: `failed: ${string}`;
|
|
130
|
+
} | {
|
|
131
|
+
readonly segment: string;
|
|
132
|
+
readonly namespace?: string;
|
|
133
|
+
/** `dryRun` only — what a real sweep would have retired, with `dropSegment`'s own preview attached. */
|
|
134
|
+
readonly action: 'would-retire';
|
|
135
|
+
readonly expiresAt: number;
|
|
136
|
+
readonly result: DropResult;
|
|
137
|
+
} | {
|
|
138
|
+
readonly segment: string;
|
|
139
|
+
readonly namespace?: string;
|
|
140
|
+
/** A tombstone row from an earlier retirement was deleted (or would be, under `dryRun`). */
|
|
141
|
+
readonly action: 'purged-tombstone' | 'would-purge-tombstone';
|
|
142
|
+
} | {
|
|
143
|
+
readonly segment: string;
|
|
144
|
+
readonly namespace?: string;
|
|
145
|
+
readonly action: 'skipped';
|
|
146
|
+
/**
|
|
147
|
+
* `'invalid-policy'` — the row has an `expiresAt` that is not usable (hand-edited, or restored from another
|
|
148
|
+
* schema). Reported rather than ignored: reading as "never expires" on a segment someone believes is
|
|
149
|
+
* expiring is the silence that costs a retention commitment.
|
|
150
|
+
* `'limit'` — eligible, but this cycle's `limit` was already spent. Re-run to continue.
|
|
151
|
+
* `'tombstone-not-empty'` — a tombstone whose Storage generations are not gone even after a GC
|
|
152
|
+
* attempt, so its row is kept: the row is what keeps the segment reachable by the generation collection and
|
|
153
|
+
* refused by every writer. Several causes, all self-healing: the storage really could not be reclaimed,
|
|
154
|
+
* the collection *declined* because the row changed underneath it (`WriteConflictError`, which this
|
|
155
|
+
* sweep swallows deliberately), or this was a `dryRun`, which reports the reason without attempting the
|
|
156
|
+
* collection at all. Except under `dryRun` the next cycle retries.
|
|
157
|
+
* `'policy-changed'` — the live row no longer says "expired" (a `clearRetention`, a new `expiresAt`, or
|
|
158
|
+
* someone else's drop landed between the enumeration and this segment's turn). Not an error: the sweep
|
|
159
|
+
* re-reads before every deletion precisely so cancelling an expiry works on a sweep already in flight.
|
|
160
|
+
* `` `failed: …` `` — any fault, isolated to this segment. A fault that happened *after* the tombstone
|
|
161
|
+
* landed is reported as `retired` with a `fault` instead, because that segment IS retired.
|
|
162
|
+
*/
|
|
163
|
+
readonly reason: 'invalid-policy' | 'limit' | 'tombstone-not-empty' | 'policy-changed' | `failed: ${string}`;
|
|
164
|
+
};
|
|
165
|
+
export interface RetireExpiredResult {
|
|
166
|
+
/** Registry rows enumerated. */
|
|
167
|
+
readonly scanned: number;
|
|
168
|
+
/** Rows whose policy said "expired" — including any the `limit` deferred. */
|
|
169
|
+
readonly eligible: number;
|
|
170
|
+
/**
|
|
171
|
+
* Segments **actually retired**. Zero under `dryRun` — see `wouldRetire`. Kept honest because this is the field
|
|
172
|
+
* most likely to end up on a dashboard: a counter that means "deleted" in one mode and
|
|
173
|
+
* "would delete" in another produces phantom deletions on any graph that does not also join on `dryRun`.
|
|
174
|
+
*/
|
|
175
|
+
readonly retired: number;
|
|
176
|
+
/** Segments a real sweep **would** have retired. Only ever non-zero under `dryRun`. */
|
|
177
|
+
readonly wouldRetire: number;
|
|
178
|
+
/** Tombstone rows actually deleted. Zero under `dryRun`; the `would-purge-tombstone` entries carry the preview. */
|
|
179
|
+
readonly tombstonesPurged: number;
|
|
180
|
+
/** True when `limit` cut the cycle short — **more segments are still eligible**. Re-run. */
|
|
181
|
+
readonly limited: boolean;
|
|
182
|
+
readonly dryRun: boolean;
|
|
183
|
+
/** Per-segment ledger. Inspect it: a `skipped` or failed entry is a segment that still holds data. */
|
|
184
|
+
readonly entries: readonly RetireEntry[];
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* How many past buckets an index scan reads besides the current one. A week: long enough that a weekend outage
|
|
188
|
+
* or a paused schedule recovers on its own, short enough that a cycle is eight list calls rather than hundreds.
|
|
189
|
+
*/
|
|
190
|
+
export declare const DEFAULT_LOOKBACK_BUCKETS = 7;
|
|
191
|
+
/**
|
|
192
|
+
* Retire every segment whose retention policy has expired, and clean up the tombstones earlier sweeps left.
|
|
193
|
+
*
|
|
194
|
+
* Returns a ledger; **never throws for a per-segment fault** (those become entries). It does throw for a bad
|
|
195
|
+
* argument, and for a fleet larger than `maxScanSegments` — a scan that cannot be held in memory is a fail-loud
|
|
196
|
+
* condition, not a partial result to be mistaken for a complete sweep.
|
|
197
|
+
*/
|
|
198
|
+
export declare function retireExpired(deps: DropDeps, options: RetireExpiredOptions): Promise<RetireExpiredResult>;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { GovernanceMeta, IRegistryDriver, SegmentRef } from './ports.js';
|
|
2
|
+
/**
|
|
3
|
+
* The smallest accepted `expiresAt`: 2001-09-09, the point below which an epoch-**seconds** value would land.
|
|
4
|
+
*
|
|
5
|
+
* This is a deliberate guard against the single most damaging typo available here. `Date.now() / 1000 + 30 * 86400`
|
|
6
|
+
* is a natural thing to write, and it produces a number ~1.8 billion — an instant in **1970**, which is to say
|
|
7
|
+
* *already expired*. Without this check, that typo is not a validation error, it is a sweep that retires the
|
|
8
|
+
* segment on its very next pass. Anything below this floor is far likelier to be seconds than a genuine intent to
|
|
9
|
+
* expire a segment two decades ago, and the caller who really means "retire it now" can say so with `dropSegment`.
|
|
10
|
+
*/
|
|
11
|
+
export declare const MIN_EXPIRES_AT_MS = 1000000000000;
|
|
12
|
+
/** A segment's retention policy: the instant it becomes eligible for retirement. */
|
|
13
|
+
export interface RetentionPolicy {
|
|
14
|
+
/**
|
|
15
|
+
* Absolute epoch-**milliseconds**. Once `now >= expiresAt`, a retention sweep may retire the segment — tombstone
|
|
16
|
+
* it and delete its Storage generations. A value in the past is legal (backfilling a policy onto existing
|
|
17
|
+
* buckets is a normal thing to do) and means "eligible on the next sweep".
|
|
18
|
+
*/
|
|
19
|
+
readonly expiresAt: number;
|
|
20
|
+
}
|
|
21
|
+
export interface RetentionDeps {
|
|
22
|
+
readonly registry: IRegistryDriver;
|
|
23
|
+
}
|
|
24
|
+
export interface SetRetentionResult {
|
|
25
|
+
readonly segment: string;
|
|
26
|
+
readonly namespace?: string;
|
|
27
|
+
/** The policy now stored on the row. */
|
|
28
|
+
readonly expiresAt: number;
|
|
29
|
+
/**
|
|
30
|
+
* True iff this call **minted the registry row** — a policy recorded before the segment's first load. The
|
|
31
|
+
* segment is now enumerable by `registry.list()` and therefore by every fleet-wide operation; the row claims no
|
|
32
|
+
* Storage generation (`currentGen: null`), so reads answer empty until the first load publishes onto it.
|
|
33
|
+
*/
|
|
34
|
+
readonly createdRow: boolean;
|
|
35
|
+
/**
|
|
36
|
+
* True iff the **due-index pointer** for this policy was written, so a fast sweep will find this segment by
|
|
37
|
+
* reading only its expiry day instead of scanning the fleet.
|
|
38
|
+
*
|
|
39
|
+
* `false` is a **degradation, not an error**, and the policy is committed either way: either the ref is too
|
|
40
|
+
* long to encode into one index-row name, or the pointer write failed. The full-scan repair pass still sees
|
|
41
|
+
* the segment's own row, so it is retired on the repair cadence rather than the fast one — slower, never
|
|
42
|
+
* never. Alarm on a *sustained* run of `false`, not on one.
|
|
43
|
+
*/
|
|
44
|
+
readonly indexed: boolean;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Read the policy off a registry row's `retention` metadata. Returns `null` when there is no policy, and
|
|
48
|
+
* `'invalid'` when the key is present but unusable (wrong type, or below the epoch-seconds floor).
|
|
49
|
+
*
|
|
50
|
+
* The three-way answer exists so a fleet sweep can *report* a malformed policy instead of choosing between two
|
|
51
|
+
* bad options: treating it as "no policy" hides an operator error on a segment someone believes is expiring, and
|
|
52
|
+
* throwing aborts a ledger over one bad row. Nothing writes an invalid policy through
|
|
53
|
+
* {@link setSegmentRetention} — hand-edited rows, an older/newer writer, or a restore are how it happens.
|
|
54
|
+
*/
|
|
55
|
+
export declare function readRetentionPolicy(meta: GovernanceMeta | undefined): RetentionPolicy | null | 'invalid';
|
|
56
|
+
/** The stored policy for one segment, or `null` if it has none (and `'invalid'` for a malformed one). */
|
|
57
|
+
export declare function getSegmentRetention(ref: SegmentRef, deps: RetentionDeps): Promise<RetentionPolicy | null | 'invalid'>;
|
|
58
|
+
export declare function setSegmentRetention(ref: SegmentRef, deps: RetentionDeps, policy: RetentionPolicy): Promise<SetRetentionResult>;
|
|
59
|
+
/**
|
|
60
|
+
* Remove a segment's expiry so no sweep will retire it. Returns whether a policy was actually removed (`false`
|
|
61
|
+
* for a segment with no row, or one that had no policy) — a separate verb from setting one, because "cancel the
|
|
62
|
+
* expiry" and "expire at time T" are different intents and a magic sentinel value for the first is how a typo
|
|
63
|
+
* becomes a deletion.
|
|
64
|
+
*
|
|
65
|
+
* Other `retention` keys are left in place; if removing the expiry empties the metadata, the field is cleared.
|
|
66
|
+
*/
|
|
67
|
+
export declare function clearSegmentRetention(ref: SegmentRef, deps: RetentionDeps): Promise<boolean>;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `withRetry` — the one retry/backoff primitive.
|
|
3
|
+
*
|
|
4
|
+
* A pure, storage-agnostic helper: it knows nothing about HTTP, AWS, or any SDK. It retries an operation
|
|
5
|
+
* while the operation keeps failing with a **retryable** error (by default {@link TransientError}), waiting
|
|
6
|
+
* a bounded, jittered, exponentially-growing delay between attempts. Time and randomness are **injected**
|
|
7
|
+
* (the determinism seam) — production wires a `setTimeout`-backed clock + a real RNG; the
|
|
8
|
+
* tests wire virtual ones — so a backoff schedule is replayable from a seed and unit tests run with no
|
|
9
|
+
* real sleeping.
|
|
10
|
+
*
|
|
11
|
+
* It deliberately does **not** retry deterministic failures ({@link WriteConflictError},
|
|
12
|
+
* {@link ValidationError}, {@link IntegrityError}, {@link NotFoundError}, …): retrying those either can't
|
|
13
|
+
* help or would be incorrect. A pointer conflict is retried by a *separate* loop inside
|
|
14
|
+
* the publish, because each attempt must re-read the row and re-decide — a blind replay of the same
|
|
15
|
+
* compare-and-swap would either fail again on a stale token or, worse, advance a pointer whose state has
|
|
16
|
+
* changed underneath it. A write-once object collision is never replayed at all: the generation number is
|
|
17
|
+
* taken, so the caller has to pick a new one.
|
|
18
|
+
*/
|
|
19
|
+
import type { Clock, Rng } from './determinism.js';
|
|
20
|
+
export interface RetryPolicy {
|
|
21
|
+
/** Total attempts including the first (so `maxAttempts: 4` = 1 try + 3 retries). Must be ≥ 1. */
|
|
22
|
+
readonly maxAttempts: number;
|
|
23
|
+
/** Delay before the first retry, in ms. Grows by `backoffFactor` each subsequent retry. */
|
|
24
|
+
readonly baseDelayMs: number;
|
|
25
|
+
/** Upper bound on a single (pre-jitter) delay, in ms — caps the exponential growth. */
|
|
26
|
+
readonly maxDelayMs: number;
|
|
27
|
+
/** Exponential growth factor between retries (e.g. 2 ⇒ base, 2·base, 4·base, …). */
|
|
28
|
+
readonly backoffFactor: number;
|
|
29
|
+
/**
|
|
30
|
+
* `'full'` ⇒ the actual wait is uniform in `[0, computed]` (AWS-recommended full jitter: decorrelates
|
|
31
|
+
* concurrent retriers so they don't reconverge into a thundering herd). `'none'` ⇒ wait exactly `computed`.
|
|
32
|
+
*/
|
|
33
|
+
readonly jitter: 'full' | 'none';
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Conservative defaults: 4 attempts, 50ms → 100 → 200 (×2), capped at 2s, full jitter. Tuned for a cloud
|
|
37
|
+
* backend's brief throttle/5xx blip — enough to ride out a transient fault without turning a hard outage
|
|
38
|
+
* into a long hang. Override per store (`retry`) or per read source (`RetryingStorageChunkSource`) if your latency
|
|
39
|
+
* budget differs.
|
|
40
|
+
*/
|
|
41
|
+
export declare const DEFAULT_RETRY_POLICY: RetryPolicy;
|
|
42
|
+
export interface RetryDeps {
|
|
43
|
+
readonly clock: Clock;
|
|
44
|
+
readonly rng: Rng;
|
|
45
|
+
/** Override which errors are retryable. Default: any {@link TransientError}. */
|
|
46
|
+
readonly isRetryable?: (err: unknown) => boolean;
|
|
47
|
+
/**
|
|
48
|
+
* Optional hook (observability) fired before each backoff wait. `attempt` is 1-based (the one that failed).
|
|
49
|
+
* Best-effort: an error it throws is swallowed, so it can neither stop the retry nor change the error thrown.
|
|
50
|
+
*/
|
|
51
|
+
readonly onRetry?: (info: {
|
|
52
|
+
attempt: number;
|
|
53
|
+
delayMs: number;
|
|
54
|
+
err: unknown;
|
|
55
|
+
}) => void;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Backoff delay (ms) for the retry that follows a given 1-based attempt, before jitter is applied. Exposed
|
|
59
|
+
* for tests. `attempt` 1 ⇒ `baseDelayMs`, 2 ⇒ `base·factor`, … capped at `maxDelayMs`.
|
|
60
|
+
*/
|
|
61
|
+
export declare function backoffDelayMs(policy: RetryPolicy, attempt: number): number;
|
|
62
|
+
/** Apply the policy's jitter to a computed delay using the injected RNG. */
|
|
63
|
+
export declare function applyJitter(policy: RetryPolicy, delayMs: number, rng: Rng): number;
|
|
64
|
+
/**
|
|
65
|
+
* Run `op`, retrying transient failures per `policy`. Resolves with `op`'s result, or rejects with the last
|
|
66
|
+
* error once attempts are exhausted (or immediately for a non-retryable error). The thrown error is always
|
|
67
|
+
* the *operation's* error — never a wrapper — so callers keep their typed-error branching.
|
|
68
|
+
*/
|
|
69
|
+
export declare function withRetry<T>(op: () => Promise<T>, policy: RetryPolicy, deps: RetryDeps): Promise<T>;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `listGenerations` and `rollbackSegment` — see what a segment has been, and put it back.
|
|
3
|
+
*
|
|
4
|
+
* Immutable generations mean the previous version of a segment is usually still sitting in the bucket: a load
|
|
5
|
+
* that replaced it did not overwrite anything, it wrote a new object and moved a pointer. So recovering from a
|
|
6
|
+
* bad load is, in principle, moving the pointer back, which no load, publish or sweep can do, because every other
|
|
7
|
+
* write path in the library only ever advances the pointer, and refuses a move that would lower it.
|
|
8
|
+
*
|
|
9
|
+
* That refusal is right for a *writer*: a load whose ids came from upstream loses nothing by being out-raced, and
|
|
10
|
+
* letting it regress the pointer would let a slow loader silently undo a fast one. It is wrong for an *operator*,
|
|
11
|
+
* who has looked at the segment, decided the current generation is wrong, and knows which one they want. So
|
|
12
|
+
* rollback is the one call that moves the pointer backwards, and it is deliberately not reachable from any
|
|
13
|
+
* automatic path — no sweep, no retry, no reconciliation calls it.
|
|
14
|
+
*
|
|
15
|
+
* It is audited for the same reason. Every other pointer move can be reconstructed from "a load happened"; this
|
|
16
|
+
* one is a human overriding the ordering rule the rest of the system relies on, which is exactly the event an
|
|
17
|
+
* Art. 30 record or an incident review wants to find.
|
|
18
|
+
*/
|
|
19
|
+
import { type IAuditSink } from './audit.js';
|
|
20
|
+
import type { IStorageDriver, IRegistryDriver, SegmentRef } from './ports.js';
|
|
21
|
+
/** What the generation helpers need: the objects, and the pointer that says which one is current. */
|
|
22
|
+
export interface GenerationListDeps {
|
|
23
|
+
readonly storage: IStorageDriver;
|
|
24
|
+
readonly registry: IRegistryDriver;
|
|
25
|
+
}
|
|
26
|
+
/** One generation present in the bucket. */
|
|
27
|
+
export interface GenerationEntry {
|
|
28
|
+
readonly generation: number;
|
|
29
|
+
/** Whether the registry pointer currently names this generation. */
|
|
30
|
+
readonly current: boolean;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Every generation still present for a segment, ascending, with the current one marked.
|
|
34
|
+
*
|
|
35
|
+
* This is what the bucket holds, not what the segment has ever been: generation collection deletes superseded
|
|
36
|
+
* objects, so the list is the grace window plus whatever has not been collected yet. It is the set a
|
|
37
|
+
* {@link rollbackSegment} can choose from, which is the reason to look at it.
|
|
38
|
+
*
|
|
39
|
+
* One registry read and one `list` call, whether or not the segment has a registry row, so it also finds the
|
|
40
|
+
* objects a purged row left behind. It does not open the objects, so it costs nothing per generation and tells you
|
|
41
|
+
* nothing about their contents — read the sizes through the store's own report if you need them.
|
|
42
|
+
*/
|
|
43
|
+
export declare function listGenerations(ref: SegmentRef, deps: GenerationListDeps): Promise<GenerationEntry[]>;
|
|
44
|
+
export interface RollbackResult {
|
|
45
|
+
/** The generation the pointer named before the rollback. `null` when the segment had no current generation. */
|
|
46
|
+
readonly fromGeneration: number | null;
|
|
47
|
+
/** The generation the pointer names now. */
|
|
48
|
+
readonly generation: number;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Move a segment's pointer **back** to a generation that is still in the bucket.
|
|
52
|
+
*
|
|
53
|
+
* The only non-forward-only publish in the library, and the only one an operator has to ask for by name. It
|
|
54
|
+
* refuses rather than guesses:
|
|
55
|
+
*
|
|
56
|
+
* - a generation whose object is **not in the bucket** — collected, or never written — is refused with
|
|
57
|
+
* `NotFoundError` rather than pointed at, because a pointer naming a missing object is the one state the whole
|
|
58
|
+
* design exists to avoid;
|
|
59
|
+
* - a **crypto-shredded** segment is refused: every generation of it is unreadable, so a rollback would produce a
|
|
60
|
+
* segment that resolves and then fails;
|
|
61
|
+
* - rolling to the generation **already current** is a no-op that reports itself, not an error.
|
|
62
|
+
*
|
|
63
|
+
* What it does **not** do is delete anything. The generations above the new pointer stay in the bucket, which is
|
|
64
|
+
* what makes the rollback reversible — roll forward again by naming one of them. They are also then *above*
|
|
65
|
+
* `currentGen`, where generation collection never looks. They stay until a load numbers above them (collection
|
|
66
|
+
* then keeps the newest `keep` of what is below its pointer), `dropSegment` deletes them, or a subject erasure
|
|
67
|
+
* deletes them: all of those present when it rewrites, and only those holding the id when the current generation does not.
|
|
68
|
+
* Not collecting them is a deliberate trade: an operator who has just undone a bad load should not have the
|
|
69
|
+
* evidence collected out from under them, while a rollback target that still holds erased data would make the
|
|
70
|
+
* erasure undoable by an ordinary operator action.
|
|
71
|
+
*/
|
|
72
|
+
export declare function rollbackSegment(ref: SegmentRef, toGeneration: number, deps: GenerationListDeps, options?: {
|
|
73
|
+
audit?: IAuditSink;
|
|
74
|
+
allowForward?: boolean;
|
|
75
|
+
}): Promise<RollbackResult>;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { ChunkRef, SegmentRef } from './ports.js';
|
|
2
|
+
/**
|
|
3
|
+
* The most an encoded name may occupy in a physical key.
|
|
4
|
+
*
|
|
5
|
+
* S3's limit is 1024 bytes for the WHOLE key, which also carries the caller's prefix, the namespace, a fixed
|
|
6
|
+
* infix (`/segments/`) and the `.<generation>.crbm` suffix. 256 leaves generous room for all of that with both
|
|
7
|
+
* a namespace and a segment at the ceiling.
|
|
8
|
+
*/
|
|
9
|
+
export declare const MAX_NAME_LENGTH = 256;
|
|
10
|
+
/**
|
|
11
|
+
* How long a name is once storage has it — the longer of the two encodings.
|
|
12
|
+
*
|
|
13
|
+
* Exported because anything that composes a name out of other names (the due index does) has to measure it the
|
|
14
|
+
* same way, and a second copy of that rule is how the two drifted apart before.
|
|
15
|
+
*/
|
|
16
|
+
export declare function encodedNameLength(value: string): number;
|
|
17
|
+
export declare function validateSegmentRef(ref: SegmentRef): void;
|
|
18
|
+
/**
|
|
19
|
+
* The namespace prefix the library keeps for its own bookkeeping: the due index stores one pointer row per
|
|
20
|
+
* expiring segment in `cbm.due.<bucket>` (see `due-index.ts`). **One constant, read by both sides** — the
|
|
21
|
+
* application-facing checks below refuse it in a name an application passes in, and the due index writes and
|
|
22
|
+
* recognises its rows by it — so the two cannot drift.
|
|
23
|
+
*
|
|
24
|
+
* Only this exact prefix is reserved, not `cbm.` as a whole: `cbm.dueX`, `cbm.due` and `cbmdue.eu` are ordinary
|
|
25
|
+
* namespaces. Every fleet-wide enumeration skips the rows of a reserved namespace as bookkeeping, so a segment
|
|
26
|
+
* there would be invisible to an erasure, a consistency check, an export and a retention sweep.
|
|
27
|
+
*
|
|
28
|
+
* Internal: not exported from any entry. The flavor keeps a copy for its own facade checks, and a test pins the
|
|
29
|
+
* two together.
|
|
30
|
+
*/
|
|
31
|
+
export declare const RESERVED_NAMESPACE_PREFIX = "cbm.due.";
|
|
32
|
+
/** Does this namespace start with {@link RESERVED_NAMESPACE_PREFIX}? `undefined` (no namespace) never does. */
|
|
33
|
+
export declare function isReservedNamespace(namespace: string | undefined): boolean;
|
|
34
|
+
/**
|
|
35
|
+
* Validate a namespace an application names, as a segment ref's namespace or as a `namespace` option that scopes
|
|
36
|
+
* a scan: the name rules, and **not** in the reserved namespace. Internal; the public `validateSegmentRef` of
|
|
37
|
+
* `@cloudbitmaps/core/driver-kit` is the name rules only, which is what a driver needs, since the due index's own
|
|
38
|
+
* rows live in the reserved namespace and go through the drivers.
|
|
39
|
+
*/
|
|
40
|
+
export declare function validateUserNamespace(namespace: string): void;
|
|
41
|
+
/** Validate a segment ref an application hands in: the name rules, and a namespace outside the reserved prefix. */
|
|
42
|
+
export declare function validateUserRef(ref: SegmentRef): void;
|
|
43
|
+
/** Validate a chunk ref: the segment/namespace rules plus `chunkKey` ∈ `[0, 65535]` (a u16). A driver-boundary check. */
|
|
44
|
+
export declare function validateChunkRef(ref: ChunkRef): void;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export type { GenKey, IRegistryDriver, IStorageDriver, SegmentRef, StorageBackend, StorageCaps, } from './core/ports.js';
|
|
2
|
+
export { brandAsBackend, STORAGE_BACKEND } from './core/ports.js';
|
|
3
|
+
export { IntegrityError, isNotFoundError, isValidationError, isWriteConflictError, NotFoundError, TransientError, ValidationError, WriteConflictError, } from './core/errors.js';
|
|
4
|
+
export { validateSegmentRef } from './core/validate.js';
|
|
5
|
+
export type { BlobSink } from './core/blob.js';
|
|
6
|
+
export { MAX_ROW_BYTES, ObjectStoreRegistry, ObjectVersionRaced, } from './drivers/_shared/object-registry.js';
|
|
7
|
+
export type { ObjectRegistryStore, ObjectRow } from './drivers/_shared/object-registry.js';
|
|
8
|
+
export { encodeNameForKey, encodeNameForPath, namespaceKeyPart, namespacePathPart, } from './drivers/_shared/keys.js';
|
|
9
|
+
export { normalizeObjectPrefix, prefixPart } from './drivers/_shared/object-registry-keys.js';
|