@cloudbitmaps/core 0.8.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/azure/index.d.cts +1 -1
- package/dist/azure/index.d.ts +1 -1
- package/dist/cassandra/index.cjs.map +1 -1
- package/dist/cassandra/index.d.cts +1 -1
- package/dist/cassandra/index.d.ts +1 -1
- package/dist/cassandra/index.js +1 -1
- package/dist/{chunk-L54FL5DW.js → chunk-BD22WDAN.js} +34 -7
- package/dist/chunk-BD22WDAN.js.map +1 -0
- package/dist/{chunk-YEHMG6W2.js → chunk-OZ6LAFV4.js} +2 -2
- package/dist/chunk-OZ6LAFV4.js.map +1 -0
- package/dist/dynamodb/index.cjs +32 -5
- package/dist/dynamodb/index.cjs.map +1 -1
- package/dist/dynamodb/index.d.cts +1 -1
- package/dist/dynamodb/index.d.ts +1 -1
- package/dist/dynamodb/index.js +2 -2
- package/dist/gcs/index.d.cts +1 -1
- package/dist/gcs/index.d.ts +1 -1
- package/dist/index.cjs +417 -35
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +244 -18
- package/dist/index.d.ts +244 -18
- package/dist/index.js +379 -34
- package/dist/index.js.map +1 -1
- package/dist/mongodb/index.cjs.map +1 -1
- package/dist/mongodb/index.d.cts +1 -1
- package/dist/mongodb/index.d.ts +1 -1
- package/dist/mongodb/index.js +1 -1
- package/dist/mysql/index.cjs.map +1 -1
- package/dist/mysql/index.d.cts +1 -1
- package/dist/mysql/index.d.ts +1 -1
- package/dist/mysql/index.js +1 -1
- package/dist/{ports-5BQTGKze.d.cts → ports-CvcvJWf6.d.cts} +46 -7
- package/dist/{ports-5BQTGKze.d.ts → ports-CvcvJWf6.d.ts} +46 -7
- package/dist/postgres/index.cjs.map +1 -1
- package/dist/postgres/index.d.cts +1 -1
- package/dist/postgres/index.d.ts +1 -1
- package/dist/postgres/index.js +1 -1
- package/dist/redis/index.cjs.map +1 -1
- package/dist/redis/index.d.cts +1 -1
- package/dist/redis/index.d.ts +1 -1
- package/dist/redis/index.js +1 -1
- package/dist/s3/index.cjs +32 -5
- package/dist/s3/index.cjs.map +1 -1
- package/dist/s3/index.d.cts +1 -1
- package/dist/s3/index.d.ts +1 -1
- package/dist/s3/index.js +1 -1
- package/package.json +1 -1
- package/dist/chunk-L54FL5DW.js.map +0 -1
- package/dist/chunk-YEHMG6W2.js.map +0 -1
package/dist/index.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { a as IWarmDriver, h as ColdChunkSource, S as SegmentRef, W as WarmReadOptions, i as SegmentSize, I as IColdDriver, d as IRegistryDriver, j as IKeystore, b as ChunkRef, C as ColdCaps, G as GenKey, B as BlobSink, R as RegCaps, e as RegistryRecord, f as NewRegistryRecord, T as Token, g as RegistryPatch, c as WarmRow, N as NoRow, k as CrbmCrypto, l as BlobReader, m as WrappedDek, A as Aead, n as AeadSealed } from './ports-
|
|
2
|
-
export {
|
|
1
|
+
import { a as IWarmDriver, h as ColdChunkSource, S as SegmentRef, W as WarmReadOptions, i as SegmentSize, I as IColdDriver, d as IRegistryDriver, j as IKeystore, b as ChunkRef, C as ColdCaps, G as GenKey, B as BlobSink, R as RegCaps, e as RegistryRecord, f as NewRegistryRecord, T as Token, g as RegistryPatch, c as WarmRow, N as NoRow, k as CrbmCrypto, l as BlobReader, m as WrappedDek, A as Aead, n as AeadSealed, o as GovernanceMeta } from './ports-CvcvJWf6.cjs';
|
|
2
|
+
export { p as BufferReader, q as BufferSink, r as NO_ROW, s as RegistryStatus, t as aadFor } from './ports-CvcvJWf6.cjs';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* The **bitmap-codec seam**.
|
|
@@ -2209,7 +2209,10 @@ interface DropResult {
|
|
|
2209
2209
|
* How the segment got to its current state. Absent on a fresh, ordinary drop (tombstone written, Cold swept).
|
|
2210
2210
|
*
|
|
2211
2211
|
* - `'warm-only'` — **a successful retirement.** No registry row and no Cold objects, so this was an accumulator
|
|
2212
|
-
* segment: its Warm rows were deleted and it now reads empty. Pairs with `dropped: true`.
|
|
2212
|
+
* segment: its Warm rows were deleted and it now reads empty. Pairs with `dropped: true`. Note this depends on
|
|
2213
|
+
* the segment having **no row**: `setRetention` mints one (that is what makes a warm-only segment enumerable),
|
|
2214
|
+
* so a retention-managed accumulator takes the ordinary row-bearing path and reports `reason: undefined` with a
|
|
2215
|
+
* tombstone instead. Branch on `dropped`; `reason` is for understanding, not for control flow.
|
|
2213
2216
|
* - `'already'` — it was already a tombstone. Idempotent; pairs with `dropped: true`. Note a re-drop is **not**
|
|
2214
2217
|
* a no-op — it also clears Warm rows that landed after the tombstone.
|
|
2215
2218
|
* - `'absent'` — **nothing existed.** No registry row, no Cold objects, and no Warm rows to delete. Pairs with
|
|
@@ -2289,6 +2292,243 @@ declare function dropSegment(ref: SegmentRef, deps: DropDeps, options: {
|
|
|
2289
2292
|
audit?: IAuditSink;
|
|
2290
2293
|
}): Promise<DropResult>;
|
|
2291
2294
|
|
|
2295
|
+
/**
|
|
2296
|
+
* The smallest accepted `expiresAt`: 2001-09-09, the point below which an epoch-**seconds** value would land.
|
|
2297
|
+
*
|
|
2298
|
+
* This is a deliberate guard against the single most damaging typo available here. `Date.now() / 1000 + 30 * 86400`
|
|
2299
|
+
* is a natural thing to write, and it produces a number ~1.8 billion — an instant in **1970**, which is to say
|
|
2300
|
+
* *already expired*. Without this check, that typo is not a validation error, it is a sweep that retires the
|
|
2301
|
+
* segment on its very next pass. Anything below this floor is far likelier to be seconds than a genuine intent to
|
|
2302
|
+
* expire a segment two decades ago, and the caller who really means "retire it now" can say so with `dropSegment`.
|
|
2303
|
+
*/
|
|
2304
|
+
declare const MIN_EXPIRES_AT_MS = 1000000000000;
|
|
2305
|
+
/** A segment's retention policy: the instant it becomes eligible for retirement. */
|
|
2306
|
+
interface RetentionPolicy {
|
|
2307
|
+
/**
|
|
2308
|
+
* Absolute epoch-**milliseconds**. Once `now >= expiresAt`, a retention sweep may retire the segment — delete
|
|
2309
|
+
* its Warm rows and its Cold generations. A value in the past is legal (backfilling a policy onto existing
|
|
2310
|
+
* buckets is a normal thing to do) and means "eligible on the next sweep".
|
|
2311
|
+
*/
|
|
2312
|
+
readonly expiresAt: number;
|
|
2313
|
+
}
|
|
2314
|
+
interface RetentionDeps {
|
|
2315
|
+
readonly registry: IRegistryDriver;
|
|
2316
|
+
}
|
|
2317
|
+
interface SetRetentionResult {
|
|
2318
|
+
readonly segment: string;
|
|
2319
|
+
readonly namespace?: string;
|
|
2320
|
+
/** The policy now stored on the row. */
|
|
2321
|
+
readonly expiresAt: number;
|
|
2322
|
+
/**
|
|
2323
|
+
* True iff this call **minted the registry row**. That is the warm-only-accumulator case: the segment existed
|
|
2324
|
+
* only as Warm deltas, invisible to `registry.list()` and therefore to every fleet-wide operation, and it is
|
|
2325
|
+
* now enumerable. The row claims no Cold generation (`currentGen: null`), so reads are unaffected.
|
|
2326
|
+
*/
|
|
2327
|
+
readonly createdRow: boolean;
|
|
2328
|
+
}
|
|
2329
|
+
/**
|
|
2330
|
+
* Read the policy off a registry row's `retention` metadata. Returns `null` when there is no policy, and
|
|
2331
|
+
* `'invalid'` when the key is present but unusable (wrong type, or below the epoch-seconds floor).
|
|
2332
|
+
*
|
|
2333
|
+
* The three-way answer exists so a fleet sweep can *report* a malformed policy instead of choosing between two
|
|
2334
|
+
* bad options: treating it as "no policy" hides an operator error on a segment someone believes is expiring, and
|
|
2335
|
+
* throwing aborts a ledger over one bad row. Nothing writes an invalid policy through
|
|
2336
|
+
* {@link setSegmentRetention} — hand-edited rows, an older/newer writer, or a restore are how it happens.
|
|
2337
|
+
*/
|
|
2338
|
+
declare function readRetentionPolicy(meta: GovernanceMeta | undefined): RetentionPolicy | null | 'invalid';
|
|
2339
|
+
/** The stored policy for one segment, or `null` if it has none (and `'invalid'` for a malformed one). */
|
|
2340
|
+
declare function getSegmentRetention(ref: SegmentRef, deps: RetentionDeps): Promise<RetentionPolicy | null | 'invalid'>;
|
|
2341
|
+
/**
|
|
2342
|
+
* Record when a segment becomes eligible for retirement, creating its registry row if it has none.
|
|
2343
|
+
*
|
|
2344
|
+
* Idempotent and safe to re-run: writing the same instant twice is a no-op in effect. Other keys already in the
|
|
2345
|
+
* row's `retention` metadata are preserved — this owns one key, not the whole object.
|
|
2346
|
+
*
|
|
2347
|
+
* Refuses a `destroyed` segment: a tombstone has nothing left to retire, and putting a policy on one would make
|
|
2348
|
+
* a sweep repeatedly "retire" bytes that are already gone.
|
|
2349
|
+
*/
|
|
2350
|
+
declare function setSegmentRetention(ref: SegmentRef, deps: RetentionDeps, policy: RetentionPolicy): Promise<SetRetentionResult>;
|
|
2351
|
+
/**
|
|
2352
|
+
* Remove a segment's expiry so no sweep will retire it. Returns whether a policy was actually removed (`false`
|
|
2353
|
+
* for a segment with no row, or one that had no policy) — a separate verb from setting one, because "cancel the
|
|
2354
|
+
* expiry" and "expire at time T" are different intents and a magic sentinel value for the first is how a typo
|
|
2355
|
+
* becomes a deletion.
|
|
2356
|
+
*
|
|
2357
|
+
* Other `retention` keys are left in place; if removing the expiry empties the metadata, the field is cleared.
|
|
2358
|
+
*/
|
|
2359
|
+
declare function clearSegmentRetention(ref: SegmentRef, deps: RetentionDeps): Promise<boolean>;
|
|
2360
|
+
|
|
2361
|
+
/**
|
|
2362
|
+
* The retention **sweep** (Phase 6) — the thing that acts on the policies `setSegmentRetention` records.
|
|
2363
|
+
*
|
|
2364
|
+
* `retireExpired` enumerates the registry, selects the segments whose `expiresAt` has passed, and retires each
|
|
2365
|
+
* one through {@link dropSegment}. It deliberately **delegates rather than reimplements**: the Warm → registry →
|
|
2366
|
+
* Cold ordering, the re-sweep for a generation staged by an in-flight compaction, and the `generationsRemaining`
|
|
2367
|
+
* report are all load-bearing and already live there. A sweep that open-coded the deletions would be a second
|
|
2368
|
+
* implementation of the most dangerous ordering in the library.
|
|
2369
|
+
*
|
|
2370
|
+
* **This is a call, not a daemon.** Nothing here schedules itself. You run it from whatever heartbeat your
|
|
2371
|
+
* deployment already has — an EventBridge rule, a Kubernetes CronJob, a queue consumer, the compaction worker's
|
|
2372
|
+
* loop — and the library stays a library: it has to behave identically in a Lambda, an edge isolate and a
|
|
2373
|
+
* long-lived server, and a timer that only works in one of those is worse than none.
|
|
2374
|
+
*
|
|
2375
|
+
* Three properties make it safe to point at a fleet:
|
|
2376
|
+
*
|
|
2377
|
+
* - **`dryRun` at the sweep level.** `dropSegment`'s `confirmSegment` guard is vacuous in a loop (it is the same
|
|
2378
|
+
* variable twice), so the meaningful preview is here: it reports what it *would* retire, per segment, and
|
|
2379
|
+
* touches nothing.
|
|
2380
|
+
* - **A per-cycle `limit`.** Clock skew, a bad backfill, or a namespace-wide policy mistake should cost one
|
|
2381
|
+
* bounded batch, not the fleet. When the limit bites, the result says so (`limited: true`) rather than looking
|
|
2382
|
+
* like a completed sweep.
|
|
2383
|
+
* - **A ledger, never an exception.** A per-segment fault is recorded and the sweep continues — the caller's
|
|
2384
|
+
* load-bearing question is *which* segments were retired, and a throw from the middle of the loop answers it
|
|
2385
|
+
* for none of them while having already retired some. Same posture as `eraseNamespace`.
|
|
2386
|
+
*
|
|
2387
|
+
* It also **purges the tombstones its own retirements leave behind**. A retired segment that had a registry row
|
|
2388
|
+
* gets a `destroyed` row, and one dead row per retired daily bucket — or per retired dedup wave — is exactly the
|
|
2389
|
+
* registry litter `dropSegment` already refuses to create for a row-less accumulator. Purging is narrow on
|
|
2390
|
+
* purpose: only a tombstone that still carries an **expired retention policy** (so it is attributably ours, never
|
|
2391
|
+
* a GDPR crypto-shred), only after a grace period, and only once Warm and Cold are provably empty for it —
|
|
2392
|
+
* because deleting the row is what makes the name reusable and takes the segment out of reach of
|
|
2393
|
+
* `gcOrphanGenerations`.
|
|
2394
|
+
*/
|
|
2395
|
+
|
|
2396
|
+
/** Default cap on retirements per sweep — a bounded batch, so a policy mistake costs one batch, not the fleet. */
|
|
2397
|
+
declare const DEFAULT_RETIRE_LIMIT = 100;
|
|
2398
|
+
/**
|
|
2399
|
+
* Default delay before a retirement's own tombstone row is purged: 24 h.
|
|
2400
|
+
*
|
|
2401
|
+
* The row is a fence — while it exists, `publishGeneration`, `bulkLoadCrbmGeneration` and `compactSegment` all
|
|
2402
|
+
* refuse the segment, so a writer that was mid-operation when the drop landed cannot resurrect it. That window is
|
|
2403
|
+
* seconds to minutes in practice; a day of margin costs one tiny row and removes any need to reason about it.
|
|
2404
|
+
*/
|
|
2405
|
+
declare const DEFAULT_TOMBSTONE_GRACE_MS = 86400000;
|
|
2406
|
+
interface RetireExpiredOptions {
|
|
2407
|
+
/** Scope the sweep to one namespace. Omit to sweep every namespace the registry knows. */
|
|
2408
|
+
readonly namespace?: string;
|
|
2409
|
+
/**
|
|
2410
|
+
* The instant to compare policies against (epoch-ms). Required here because `core/` takes its time from the
|
|
2411
|
+
* caller, never from the platform — `store.retireExpired()` fills it in from the store's clock.
|
|
2412
|
+
*/
|
|
2413
|
+
readonly now: number;
|
|
2414
|
+
/** Maximum segments to retire in this cycle (default {@link DEFAULT_RETIRE_LIMIT}). */
|
|
2415
|
+
readonly limit?: number;
|
|
2416
|
+
/** Report what would be retired and change nothing. */
|
|
2417
|
+
readonly dryRun?: boolean;
|
|
2418
|
+
/** Forwarded to each `dropSegment`, so every retirement lands in the audit trail as `segment.dispose`. */
|
|
2419
|
+
readonly audit?: IAuditSink;
|
|
2420
|
+
/** Ceiling on rows enumerated (default `DEFAULT_MAX_SCAN_SEGMENTS`, 250,000); exceeding it throws. */
|
|
2421
|
+
readonly maxScanSegments?: number;
|
|
2422
|
+
/**
|
|
2423
|
+
* Whether to delete the tombstone rows this sweep's own past retirements left (default `true`). Set `false` to
|
|
2424
|
+
* keep every tombstone forever — the right choice if something outside this library treats the presence of a
|
|
2425
|
+
* `destroyed` row as an attestation.
|
|
2426
|
+
*
|
|
2427
|
+
* Two knobs rather than one `number | 'never'`, deliberately: `0` would have had to mean "purge immediately"
|
|
2428
|
+
* here while `coldGenTtlMs: 0` in this same library means "pin forever", and one option whose zero is the
|
|
2429
|
+
* opposite of another's is a reading hazard for whoever tunes both.
|
|
2430
|
+
*/
|
|
2431
|
+
readonly purgeTombstones?: boolean;
|
|
2432
|
+
/** How long a retirement's tombstone must age before this sweep deletes it (default 24 h). */
|
|
2433
|
+
readonly tombstoneGraceMs?: number;
|
|
2434
|
+
}
|
|
2435
|
+
/** What happened to one segment in a sweep. */
|
|
2436
|
+
type RetireEntry = {
|
|
2437
|
+
readonly segment: string;
|
|
2438
|
+
readonly namespace?: string;
|
|
2439
|
+
/** Retired: `result` is `dropSegment`'s full report — **check `generationsRemaining`**. */
|
|
2440
|
+
readonly action: 'retired';
|
|
2441
|
+
readonly expiresAt: number;
|
|
2442
|
+
readonly result: DropResult;
|
|
2443
|
+
/**
|
|
2444
|
+
* Present when the retirement **completed the destructive part and then faulted** — the tombstone is written
|
|
2445
|
+
* and the segment reads empty, but something after that (the Cold sweep) threw. The storage may not be fully
|
|
2446
|
+
* reclaimed; re-run. Reported as `retired` rather than `skipped` because the segment really is retired, and
|
|
2447
|
+
* saying otherwise is the one thing a caller cannot recover from.
|
|
2448
|
+
*/
|
|
2449
|
+
readonly fault?: `failed: ${string}`;
|
|
2450
|
+
} | {
|
|
2451
|
+
readonly segment: string;
|
|
2452
|
+
readonly namespace?: string;
|
|
2453
|
+
/** `dryRun` only — what a real sweep would have retired, with `dropSegment`'s own preview attached. */
|
|
2454
|
+
readonly action: 'would-retire';
|
|
2455
|
+
readonly expiresAt: number;
|
|
2456
|
+
readonly result: DropResult;
|
|
2457
|
+
} | {
|
|
2458
|
+
readonly segment: string;
|
|
2459
|
+
readonly namespace?: string;
|
|
2460
|
+
/** A tombstone row from an earlier retirement was deleted (or would be, under `dryRun`). */
|
|
2461
|
+
readonly action: 'purged-tombstone' | 'would-purge-tombstone';
|
|
2462
|
+
} | {
|
|
2463
|
+
readonly segment: string;
|
|
2464
|
+
readonly namespace?: string;
|
|
2465
|
+
readonly action: 'skipped';
|
|
2466
|
+
/**
|
|
2467
|
+
* `'invalid-policy'` — the row has an `expiresAt` that is not usable (hand-edited, or restored from another
|
|
2468
|
+
* schema). Reported rather than ignored: reading as "never expires" on a segment someone believes is
|
|
2469
|
+
* expiring is the silence that costs a retention commitment.
|
|
2470
|
+
* `'limit'` — eligible, but this cycle's `limit` was already spent. Re-run to continue.
|
|
2471
|
+
* `'tombstone-not-empty'` — a tombstone whose Warm rows or Cold generations are not gone even after a GC
|
|
2472
|
+
* attempt, so its row is kept: the row is what keeps the segment reachable by `gcOrphanGenerations` and
|
|
2473
|
+
* refused by every writer.
|
|
2474
|
+
* `'policy-changed'` — the live row no longer says "expired" (a `clearRetention`, a new `expiresAt`, or
|
|
2475
|
+
* someone else's drop landed between the enumeration and this segment's turn). Not an error: the sweep
|
|
2476
|
+
* re-reads before every deletion precisely so cancelling an expiry works on a sweep already in flight.
|
|
2477
|
+
* `` `failed: …` `` — any fault, isolated to this segment. A fault that happened *after* the tombstone
|
|
2478
|
+
* landed is reported as `retired` with a `fault` instead, because that segment IS retired.
|
|
2479
|
+
*/
|
|
2480
|
+
readonly reason: 'invalid-policy' | 'limit' | 'tombstone-not-empty' | 'policy-changed' | `failed: ${string}`;
|
|
2481
|
+
};
|
|
2482
|
+
interface RetireExpiredResult {
|
|
2483
|
+
/** Registry rows enumerated. */
|
|
2484
|
+
readonly scanned: number;
|
|
2485
|
+
/** Rows whose policy said "expired" — including any the `limit` deferred. */
|
|
2486
|
+
readonly eligible: number;
|
|
2487
|
+
/**
|
|
2488
|
+
* Segments **actually retired**. Zero under `dryRun` — see `wouldRetire`. Kept honest because this is the field
|
|
2489
|
+
* most likely to end up on a dashboard, and the CLI emits it: a counter that means "deleted" in one mode and
|
|
2490
|
+
* "would delete" in another produces phantom deletions on any graph that does not also join on `dryRun`.
|
|
2491
|
+
*/
|
|
2492
|
+
readonly retired: number;
|
|
2493
|
+
/** Segments a real sweep **would** have retired. Only ever non-zero under `dryRun`. */
|
|
2494
|
+
readonly wouldRetire: number;
|
|
2495
|
+
/** Tombstone rows actually deleted. Zero under `dryRun`; the `would-purge-tombstone` entries carry the preview. */
|
|
2496
|
+
readonly tombstonesPurged: number;
|
|
2497
|
+
/** True when `limit` cut the cycle short — **more segments are still eligible**. Re-run. */
|
|
2498
|
+
readonly limited: boolean;
|
|
2499
|
+
readonly dryRun: boolean;
|
|
2500
|
+
/** Per-segment ledger. Inspect it: a `skipped` or failed entry is a segment that still holds data. */
|
|
2501
|
+
readonly entries: readonly RetireEntry[];
|
|
2502
|
+
}
|
|
2503
|
+
/**
|
|
2504
|
+
* Retire every segment whose retention policy has expired, and clean up the tombstones earlier sweeps left.
|
|
2505
|
+
*
|
|
2506
|
+
* Returns a ledger; **never throws for a per-segment fault** (those become entries). It does throw for a bad
|
|
2507
|
+
* argument, and for a fleet larger than `maxScanSegments` — a scan that cannot be held in memory is a fail-loud
|
|
2508
|
+
* condition, not a partial result to be mistaken for a complete sweep.
|
|
2509
|
+
*/
|
|
2510
|
+
declare function retireExpired(deps: DropDeps, options: RetireExpiredOptions): Promise<RetireExpiredResult>;
|
|
2511
|
+
|
|
2512
|
+
/**
|
|
2513
|
+
* Ceiling on registry records one fleet-wide scan holds resident. Raise it via the caller's `maxScanSegments` when
|
|
2514
|
+
* the fleet really is that large *and* the memory is there; narrow the scan with a `namespace` otherwise.
|
|
2515
|
+
*/
|
|
2516
|
+
declare const DEFAULT_MAX_SCAN_SEGMENTS = 250000;
|
|
2517
|
+
/** Fail fast on a bad ceiling BEFORE the (possibly huge) scan, not after. */
|
|
2518
|
+
declare function validateMaxScanSegments(value: number, op: string): void;
|
|
2519
|
+
/**
|
|
2520
|
+
* Drain every registry record (optionally one namespace) into an array, refusing past `maxScanSegments`.
|
|
2521
|
+
*
|
|
2522
|
+
* The bound is checked **before** each push, so the array never exceeds the ceiling the caller agreed to — the
|
|
2523
|
+
* earlier form pushed first and then threw, which meant the very row that broke the budget was already resident,
|
|
2524
|
+
* and a caller sizing a container against `maxScanSegments` was off by one row at the worst possible moment.
|
|
2525
|
+
*/
|
|
2526
|
+
declare function drainRegistry(registry: IRegistryDriver, options: {
|
|
2527
|
+
namespace?: string;
|
|
2528
|
+
maxScanSegments: number;
|
|
2529
|
+
op: string;
|
|
2530
|
+
}): Promise<RegistryRecord[]>;
|
|
2531
|
+
|
|
2292
2532
|
/**
|
|
2293
2533
|
* Fail-safe cross-tier DR check (gap #11). The registry (`currentGen`, e.g. in
|
|
2294
2534
|
* DynamoDB) and the immutable `.crbm` generations (e.g. in S3) back up and restore **independently**, so a
|
|
@@ -2307,20 +2547,6 @@ declare function dropSegment(ref: SegmentRef, deps: DropDeps, options: {
|
|
|
2307
2547
|
* to confirm a reported tear.
|
|
2308
2548
|
*/
|
|
2309
2549
|
|
|
2310
|
-
/** Default in-flight fan-out for the consistency scan — bounded, no thundering herd. */
|
|
2311
|
-
/**
|
|
2312
|
-
* Default ceiling on how many registry records one consistency check may hold resident: 250,000.
|
|
2313
|
-
*
|
|
2314
|
-
* **Named `maxScanSegments`, not `maxSegments`, on purpose.** Compaction already has a `maxSegments`
|
|
2315
|
-
* (`CR_COMPACT_MAX_SEGMENTS`) that caps how much work one cycle does and then *continues normally*. This one
|
|
2316
|
-
* caps how much a scan may hold and *refuses* past it. Same-sounding names with opposite behaviour at the
|
|
2317
|
-
* limit is a trap; pairing this with `maxWarmScanBytes` instead makes both read as what they are — ceilings on
|
|
2318
|
-
* a scan that fail closed.
|
|
2319
|
-
*
|
|
2320
|
-
* Generous — the compaction docs target 100K+ segment fleets — while still bounding a DR drill's memory to
|
|
2321
|
-
* something a modest operator box survives. Raisable, because a ceiling you cannot lift is a landmine.
|
|
2322
|
-
*/
|
|
2323
|
-
declare const DEFAULT_MAX_SCAN_SEGMENTS = 250000;
|
|
2324
2550
|
interface ConsistencyIssue {
|
|
2325
2551
|
readonly segment: string;
|
|
2326
2552
|
readonly namespace?: string;
|
|
@@ -2532,4 +2758,4 @@ declare class CrbmWriter {
|
|
|
2532
2758
|
private buildFooter;
|
|
2533
2759
|
}
|
|
2534
2760
|
|
|
2535
|
-
export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type DropDeps, type DropResult, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, dropSegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
|
|
2761
|
+
export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETIRE_LIMIT, DEFAULT_RETRY_POLICY, DEFAULT_TOMBSTONE_GRACE_MS, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type DropDeps, type DropResult, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, GovernanceMeta, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MIN_EXPIRES_AT_MS, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetentionDeps, type RetentionPolicy, type RetireEntry, type RetireExpiredOptions, type RetireExpiredResult, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, type SetRetentionResult, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, clearSegmentRetention, collectWithinBudget, compactSegment, destroySegment, drainRegistry, dropSegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, getSegmentRetention, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, readRetentionPolicy, resolveBudget, resolvePerOpBudget, retireExpired, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, setSegmentRetention, splitId, validateCompactionOptions, validateMaxScanSegments, validateSegmentRef, withRetry, writeCrbmGeneration };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { a as IWarmDriver, h as ColdChunkSource, S as SegmentRef, W as WarmReadOptions, i as SegmentSize, I as IColdDriver, d as IRegistryDriver, j as IKeystore, b as ChunkRef, C as ColdCaps, G as GenKey, B as BlobSink, R as RegCaps, e as RegistryRecord, f as NewRegistryRecord, T as Token, g as RegistryPatch, c as WarmRow, N as NoRow, k as CrbmCrypto, l as BlobReader, m as WrappedDek, A as Aead, n as AeadSealed } from './ports-
|
|
2
|
-
export {
|
|
1
|
+
import { a as IWarmDriver, h as ColdChunkSource, S as SegmentRef, W as WarmReadOptions, i as SegmentSize, I as IColdDriver, d as IRegistryDriver, j as IKeystore, b as ChunkRef, C as ColdCaps, G as GenKey, B as BlobSink, R as RegCaps, e as RegistryRecord, f as NewRegistryRecord, T as Token, g as RegistryPatch, c as WarmRow, N as NoRow, k as CrbmCrypto, l as BlobReader, m as WrappedDek, A as Aead, n as AeadSealed, o as GovernanceMeta } from './ports-CvcvJWf6.js';
|
|
2
|
+
export { p as BufferReader, q as BufferSink, r as NO_ROW, s as RegistryStatus, t as aadFor } from './ports-CvcvJWf6.js';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* The **bitmap-codec seam**.
|
|
@@ -2209,7 +2209,10 @@ interface DropResult {
|
|
|
2209
2209
|
* How the segment got to its current state. Absent on a fresh, ordinary drop (tombstone written, Cold swept).
|
|
2210
2210
|
*
|
|
2211
2211
|
* - `'warm-only'` — **a successful retirement.** No registry row and no Cold objects, so this was an accumulator
|
|
2212
|
-
* segment: its Warm rows were deleted and it now reads empty. Pairs with `dropped: true`.
|
|
2212
|
+
* segment: its Warm rows were deleted and it now reads empty. Pairs with `dropped: true`. Note this depends on
|
|
2213
|
+
* the segment having **no row**: `setRetention` mints one (that is what makes a warm-only segment enumerable),
|
|
2214
|
+
* so a retention-managed accumulator takes the ordinary row-bearing path and reports `reason: undefined` with a
|
|
2215
|
+
* tombstone instead. Branch on `dropped`; `reason` is for understanding, not for control flow.
|
|
2213
2216
|
* - `'already'` — it was already a tombstone. Idempotent; pairs with `dropped: true`. Note a re-drop is **not**
|
|
2214
2217
|
* a no-op — it also clears Warm rows that landed after the tombstone.
|
|
2215
2218
|
* - `'absent'` — **nothing existed.** No registry row, no Cold objects, and no Warm rows to delete. Pairs with
|
|
@@ -2289,6 +2292,243 @@ declare function dropSegment(ref: SegmentRef, deps: DropDeps, options: {
|
|
|
2289
2292
|
audit?: IAuditSink;
|
|
2290
2293
|
}): Promise<DropResult>;
|
|
2291
2294
|
|
|
2295
|
+
/**
|
|
2296
|
+
* The smallest accepted `expiresAt`: 2001-09-09, the point below which an epoch-**seconds** value would land.
|
|
2297
|
+
*
|
|
2298
|
+
* This is a deliberate guard against the single most damaging typo available here. `Date.now() / 1000 + 30 * 86400`
|
|
2299
|
+
* is a natural thing to write, and it produces a number ~1.8 billion — an instant in **1970**, which is to say
|
|
2300
|
+
* *already expired*. Without this check, that typo is not a validation error, it is a sweep that retires the
|
|
2301
|
+
* segment on its very next pass. Anything below this floor is far likelier to be seconds than a genuine intent to
|
|
2302
|
+
* expire a segment two decades ago, and the caller who really means "retire it now" can say so with `dropSegment`.
|
|
2303
|
+
*/
|
|
2304
|
+
declare const MIN_EXPIRES_AT_MS = 1000000000000;
|
|
2305
|
+
/** A segment's retention policy: the instant it becomes eligible for retirement. */
|
|
2306
|
+
interface RetentionPolicy {
|
|
2307
|
+
/**
|
|
2308
|
+
* Absolute epoch-**milliseconds**. Once `now >= expiresAt`, a retention sweep may retire the segment — delete
|
|
2309
|
+
* its Warm rows and its Cold generations. A value in the past is legal (backfilling a policy onto existing
|
|
2310
|
+
* buckets is a normal thing to do) and means "eligible on the next sweep".
|
|
2311
|
+
*/
|
|
2312
|
+
readonly expiresAt: number;
|
|
2313
|
+
}
|
|
2314
|
+
interface RetentionDeps {
|
|
2315
|
+
readonly registry: IRegistryDriver;
|
|
2316
|
+
}
|
|
2317
|
+
interface SetRetentionResult {
|
|
2318
|
+
readonly segment: string;
|
|
2319
|
+
readonly namespace?: string;
|
|
2320
|
+
/** The policy now stored on the row. */
|
|
2321
|
+
readonly expiresAt: number;
|
|
2322
|
+
/**
|
|
2323
|
+
* True iff this call **minted the registry row**. That is the warm-only-accumulator case: the segment existed
|
|
2324
|
+
* only as Warm deltas, invisible to `registry.list()` and therefore to every fleet-wide operation, and it is
|
|
2325
|
+
* now enumerable. The row claims no Cold generation (`currentGen: null`), so reads are unaffected.
|
|
2326
|
+
*/
|
|
2327
|
+
readonly createdRow: boolean;
|
|
2328
|
+
}
|
|
2329
|
+
/**
|
|
2330
|
+
* Read the policy off a registry row's `retention` metadata. Returns `null` when there is no policy, and
|
|
2331
|
+
* `'invalid'` when the key is present but unusable (wrong type, or below the epoch-seconds floor).
|
|
2332
|
+
*
|
|
2333
|
+
* The three-way answer exists so a fleet sweep can *report* a malformed policy instead of choosing between two
|
|
2334
|
+
* bad options: treating it as "no policy" hides an operator error on a segment someone believes is expiring, and
|
|
2335
|
+
* throwing aborts a ledger over one bad row. Nothing writes an invalid policy through
|
|
2336
|
+
* {@link setSegmentRetention} — hand-edited rows, an older/newer writer, or a restore are how it happens.
|
|
2337
|
+
*/
|
|
2338
|
+
declare function readRetentionPolicy(meta: GovernanceMeta | undefined): RetentionPolicy | null | 'invalid';
|
|
2339
|
+
/** The stored policy for one segment, or `null` if it has none (and `'invalid'` for a malformed one). */
|
|
2340
|
+
declare function getSegmentRetention(ref: SegmentRef, deps: RetentionDeps): Promise<RetentionPolicy | null | 'invalid'>;
|
|
2341
|
+
/**
|
|
2342
|
+
* Record when a segment becomes eligible for retirement, creating its registry row if it has none.
|
|
2343
|
+
*
|
|
2344
|
+
* Idempotent and safe to re-run: writing the same instant twice is a no-op in effect. Other keys already in the
|
|
2345
|
+
* row's `retention` metadata are preserved — this owns one key, not the whole object.
|
|
2346
|
+
*
|
|
2347
|
+
* Refuses a `destroyed` segment: a tombstone has nothing left to retire, and putting a policy on one would make
|
|
2348
|
+
* a sweep repeatedly "retire" bytes that are already gone.
|
|
2349
|
+
*/
|
|
2350
|
+
declare function setSegmentRetention(ref: SegmentRef, deps: RetentionDeps, policy: RetentionPolicy): Promise<SetRetentionResult>;
|
|
2351
|
+
/**
|
|
2352
|
+
* Remove a segment's expiry so no sweep will retire it. Returns whether a policy was actually removed (`false`
|
|
2353
|
+
* for a segment with no row, or one that had no policy) — a separate verb from setting one, because "cancel the
|
|
2354
|
+
* expiry" and "expire at time T" are different intents and a magic sentinel value for the first is how a typo
|
|
2355
|
+
* becomes a deletion.
|
|
2356
|
+
*
|
|
2357
|
+
* Other `retention` keys are left in place; if removing the expiry empties the metadata, the field is cleared.
|
|
2358
|
+
*/
|
|
2359
|
+
declare function clearSegmentRetention(ref: SegmentRef, deps: RetentionDeps): Promise<boolean>;
|
|
2360
|
+
|
|
2361
|
+
/**
|
|
2362
|
+
* The retention **sweep** (Phase 6) — the thing that acts on the policies `setSegmentRetention` records.
|
|
2363
|
+
*
|
|
2364
|
+
* `retireExpired` enumerates the registry, selects the segments whose `expiresAt` has passed, and retires each
|
|
2365
|
+
* one through {@link dropSegment}. It deliberately **delegates rather than reimplements**: the Warm → registry →
|
|
2366
|
+
* Cold ordering, the re-sweep for a generation staged by an in-flight compaction, and the `generationsRemaining`
|
|
2367
|
+
* report are all load-bearing and already live there. A sweep that open-coded the deletions would be a second
|
|
2368
|
+
* implementation of the most dangerous ordering in the library.
|
|
2369
|
+
*
|
|
2370
|
+
* **This is a call, not a daemon.** Nothing here schedules itself. You run it from whatever heartbeat your
|
|
2371
|
+
* deployment already has — an EventBridge rule, a Kubernetes CronJob, a queue consumer, the compaction worker's
|
|
2372
|
+
* loop — and the library stays a library: it has to behave identically in a Lambda, an edge isolate and a
|
|
2373
|
+
* long-lived server, and a timer that only works in one of those is worse than none.
|
|
2374
|
+
*
|
|
2375
|
+
* Three properties make it safe to point at a fleet:
|
|
2376
|
+
*
|
|
2377
|
+
* - **`dryRun` at the sweep level.** `dropSegment`'s `confirmSegment` guard is vacuous in a loop (it is the same
|
|
2378
|
+
* variable twice), so the meaningful preview is here: it reports what it *would* retire, per segment, and
|
|
2379
|
+
* touches nothing.
|
|
2380
|
+
* - **A per-cycle `limit`.** Clock skew, a bad backfill, or a namespace-wide policy mistake should cost one
|
|
2381
|
+
* bounded batch, not the fleet. When the limit bites, the result says so (`limited: true`) rather than looking
|
|
2382
|
+
* like a completed sweep.
|
|
2383
|
+
* - **A ledger, never an exception.** A per-segment fault is recorded and the sweep continues — the caller's
|
|
2384
|
+
* load-bearing question is *which* segments were retired, and a throw from the middle of the loop answers it
|
|
2385
|
+
* for none of them while having already retired some. Same posture as `eraseNamespace`.
|
|
2386
|
+
*
|
|
2387
|
+
* It also **purges the tombstones its own retirements leave behind**. A retired segment that had a registry row
|
|
2388
|
+
* gets a `destroyed` row, and one dead row per retired daily bucket — or per retired dedup wave — is exactly the
|
|
2389
|
+
* registry litter `dropSegment` already refuses to create for a row-less accumulator. Purging is narrow on
|
|
2390
|
+
* purpose: only a tombstone that still carries an **expired retention policy** (so it is attributably ours, never
|
|
2391
|
+
* a GDPR crypto-shred), only after a grace period, and only once Warm and Cold are provably empty for it —
|
|
2392
|
+
* because deleting the row is what makes the name reusable and takes the segment out of reach of
|
|
2393
|
+
* `gcOrphanGenerations`.
|
|
2394
|
+
*/
|
|
2395
|
+
|
|
2396
|
+
/** Default cap on retirements per sweep — a bounded batch, so a policy mistake costs one batch, not the fleet. */
|
|
2397
|
+
declare const DEFAULT_RETIRE_LIMIT = 100;
|
|
2398
|
+
/**
|
|
2399
|
+
* Default delay before a retirement's own tombstone row is purged: 24 h.
|
|
2400
|
+
*
|
|
2401
|
+
* The row is a fence — while it exists, `publishGeneration`, `bulkLoadCrbmGeneration` and `compactSegment` all
|
|
2402
|
+
* refuse the segment, so a writer that was mid-operation when the drop landed cannot resurrect it. That window is
|
|
2403
|
+
* seconds to minutes in practice; a day of margin costs one tiny row and removes any need to reason about it.
|
|
2404
|
+
*/
|
|
2405
|
+
declare const DEFAULT_TOMBSTONE_GRACE_MS = 86400000;
|
|
2406
|
+
interface RetireExpiredOptions {
|
|
2407
|
+
/** Scope the sweep to one namespace. Omit to sweep every namespace the registry knows. */
|
|
2408
|
+
readonly namespace?: string;
|
|
2409
|
+
/**
|
|
2410
|
+
* The instant to compare policies against (epoch-ms). Required here because `core/` takes its time from the
|
|
2411
|
+
* caller, never from the platform — `store.retireExpired()` fills it in from the store's clock.
|
|
2412
|
+
*/
|
|
2413
|
+
readonly now: number;
|
|
2414
|
+
/** Maximum segments to retire in this cycle (default {@link DEFAULT_RETIRE_LIMIT}). */
|
|
2415
|
+
readonly limit?: number;
|
|
2416
|
+
/** Report what would be retired and change nothing. */
|
|
2417
|
+
readonly dryRun?: boolean;
|
|
2418
|
+
/** Forwarded to each `dropSegment`, so every retirement lands in the audit trail as `segment.dispose`. */
|
|
2419
|
+
readonly audit?: IAuditSink;
|
|
2420
|
+
/** Ceiling on rows enumerated (default `DEFAULT_MAX_SCAN_SEGMENTS`, 250,000); exceeding it throws. */
|
|
2421
|
+
readonly maxScanSegments?: number;
|
|
2422
|
+
/**
|
|
2423
|
+
* Whether to delete the tombstone rows this sweep's own past retirements left (default `true`). Set `false` to
|
|
2424
|
+
* keep every tombstone forever — the right choice if something outside this library treats the presence of a
|
|
2425
|
+
* `destroyed` row as an attestation.
|
|
2426
|
+
*
|
|
2427
|
+
* Two knobs rather than one `number | 'never'`, deliberately: `0` would have had to mean "purge immediately"
|
|
2428
|
+
* here while `coldGenTtlMs: 0` in this same library means "pin forever", and one option whose zero is the
|
|
2429
|
+
* opposite of another's is a reading hazard for whoever tunes both.
|
|
2430
|
+
*/
|
|
2431
|
+
readonly purgeTombstones?: boolean;
|
|
2432
|
+
/** How long a retirement's tombstone must age before this sweep deletes it (default 24 h). */
|
|
2433
|
+
readonly tombstoneGraceMs?: number;
|
|
2434
|
+
}
|
|
2435
|
+
/** What happened to one segment in a sweep. */
|
|
2436
|
+
type RetireEntry = {
|
|
2437
|
+
readonly segment: string;
|
|
2438
|
+
readonly namespace?: string;
|
|
2439
|
+
/** Retired: `result` is `dropSegment`'s full report — **check `generationsRemaining`**. */
|
|
2440
|
+
readonly action: 'retired';
|
|
2441
|
+
readonly expiresAt: number;
|
|
2442
|
+
readonly result: DropResult;
|
|
2443
|
+
/**
|
|
2444
|
+
* Present when the retirement **completed the destructive part and then faulted** — the tombstone is written
|
|
2445
|
+
* and the segment reads empty, but something after that (the Cold sweep) threw. The storage may not be fully
|
|
2446
|
+
* reclaimed; re-run. Reported as `retired` rather than `skipped` because the segment really is retired, and
|
|
2447
|
+
* saying otherwise is the one thing a caller cannot recover from.
|
|
2448
|
+
*/
|
|
2449
|
+
readonly fault?: `failed: ${string}`;
|
|
2450
|
+
} | {
|
|
2451
|
+
readonly segment: string;
|
|
2452
|
+
readonly namespace?: string;
|
|
2453
|
+
/** `dryRun` only — what a real sweep would have retired, with `dropSegment`'s own preview attached. */
|
|
2454
|
+
readonly action: 'would-retire';
|
|
2455
|
+
readonly expiresAt: number;
|
|
2456
|
+
readonly result: DropResult;
|
|
2457
|
+
} | {
|
|
2458
|
+
readonly segment: string;
|
|
2459
|
+
readonly namespace?: string;
|
|
2460
|
+
/** A tombstone row from an earlier retirement was deleted (or would be, under `dryRun`). */
|
|
2461
|
+
readonly action: 'purged-tombstone' | 'would-purge-tombstone';
|
|
2462
|
+
} | {
|
|
2463
|
+
readonly segment: string;
|
|
2464
|
+
readonly namespace?: string;
|
|
2465
|
+
readonly action: 'skipped';
|
|
2466
|
+
/**
|
|
2467
|
+
* `'invalid-policy'` — the row has an `expiresAt` that is not usable (hand-edited, or restored from another
|
|
2468
|
+
* schema). Reported rather than ignored: reading as "never expires" on a segment someone believes is
|
|
2469
|
+
* expiring is the silence that costs a retention commitment.
|
|
2470
|
+
* `'limit'` — eligible, but this cycle's `limit` was already spent. Re-run to continue.
|
|
2471
|
+
* `'tombstone-not-empty'` — a tombstone whose Warm rows or Cold generations are not gone even after a GC
|
|
2472
|
+
* attempt, so its row is kept: the row is what keeps the segment reachable by `gcOrphanGenerations` and
|
|
2473
|
+
* refused by every writer.
|
|
2474
|
+
* `'policy-changed'` — the live row no longer says "expired" (a `clearRetention`, a new `expiresAt`, or
|
|
2475
|
+
* someone else's drop landed between the enumeration and this segment's turn). Not an error: the sweep
|
|
2476
|
+
* re-reads before every deletion precisely so cancelling an expiry works on a sweep already in flight.
|
|
2477
|
+
* `` `failed: …` `` — any fault, isolated to this segment. A fault that happened *after* the tombstone
|
|
2478
|
+
* landed is reported as `retired` with a `fault` instead, because that segment IS retired.
|
|
2479
|
+
*/
|
|
2480
|
+
readonly reason: 'invalid-policy' | 'limit' | 'tombstone-not-empty' | 'policy-changed' | `failed: ${string}`;
|
|
2481
|
+
};
|
|
2482
|
+
interface RetireExpiredResult {
|
|
2483
|
+
/** Registry rows enumerated. */
|
|
2484
|
+
readonly scanned: number;
|
|
2485
|
+
/** Rows whose policy said "expired" — including any the `limit` deferred. */
|
|
2486
|
+
readonly eligible: number;
|
|
2487
|
+
/**
|
|
2488
|
+
* Segments **actually retired**. Zero under `dryRun` — see `wouldRetire`. Kept honest because this is the field
|
|
2489
|
+
* most likely to end up on a dashboard, and the CLI emits it: a counter that means "deleted" in one mode and
|
|
2490
|
+
* "would delete" in another produces phantom deletions on any graph that does not also join on `dryRun`.
|
|
2491
|
+
*/
|
|
2492
|
+
readonly retired: number;
|
|
2493
|
+
/** Segments a real sweep **would** have retired. Only ever non-zero under `dryRun`. */
|
|
2494
|
+
readonly wouldRetire: number;
|
|
2495
|
+
/** Tombstone rows actually deleted. Zero under `dryRun`; the `would-purge-tombstone` entries carry the preview. */
|
|
2496
|
+
readonly tombstonesPurged: number;
|
|
2497
|
+
/** True when `limit` cut the cycle short — **more segments are still eligible**. Re-run. */
|
|
2498
|
+
readonly limited: boolean;
|
|
2499
|
+
readonly dryRun: boolean;
|
|
2500
|
+
/** Per-segment ledger. Inspect it: a `skipped` or failed entry is a segment that still holds data. */
|
|
2501
|
+
readonly entries: readonly RetireEntry[];
|
|
2502
|
+
}
|
|
2503
|
+
/**
|
|
2504
|
+
* Retire every segment whose retention policy has expired, and clean up the tombstones earlier sweeps left.
|
|
2505
|
+
*
|
|
2506
|
+
* Returns a ledger; **never throws for a per-segment fault** (those become entries). It does throw for a bad
|
|
2507
|
+
* argument, and for a fleet larger than `maxScanSegments` — a scan that cannot be held in memory is a fail-loud
|
|
2508
|
+
* condition, not a partial result to be mistaken for a complete sweep.
|
|
2509
|
+
*/
|
|
2510
|
+
declare function retireExpired(deps: DropDeps, options: RetireExpiredOptions): Promise<RetireExpiredResult>;
|
|
2511
|
+
|
|
2512
|
+
/**
|
|
2513
|
+
* Ceiling on registry records one fleet-wide scan holds resident. Raise it via the caller's `maxScanSegments` when
|
|
2514
|
+
* the fleet really is that large *and* the memory is there; narrow the scan with a `namespace` otherwise.
|
|
2515
|
+
*/
|
|
2516
|
+
declare const DEFAULT_MAX_SCAN_SEGMENTS = 250000;
|
|
2517
|
+
/** Fail fast on a bad ceiling BEFORE the (possibly huge) scan, not after. */
|
|
2518
|
+
declare function validateMaxScanSegments(value: number, op: string): void;
|
|
2519
|
+
/**
|
|
2520
|
+
* Drain every registry record (optionally one namespace) into an array, refusing past `maxScanSegments`.
|
|
2521
|
+
*
|
|
2522
|
+
* The bound is checked **before** each push, so the array never exceeds the ceiling the caller agreed to — the
|
|
2523
|
+
* earlier form pushed first and then threw, which meant the very row that broke the budget was already resident,
|
|
2524
|
+
* and a caller sizing a container against `maxScanSegments` was off by one row at the worst possible moment.
|
|
2525
|
+
*/
|
|
2526
|
+
declare function drainRegistry(registry: IRegistryDriver, options: {
|
|
2527
|
+
namespace?: string;
|
|
2528
|
+
maxScanSegments: number;
|
|
2529
|
+
op: string;
|
|
2530
|
+
}): Promise<RegistryRecord[]>;
|
|
2531
|
+
|
|
2292
2532
|
/**
|
|
2293
2533
|
* Fail-safe cross-tier DR check (gap #11). The registry (`currentGen`, e.g. in
|
|
2294
2534
|
* DynamoDB) and the immutable `.crbm` generations (e.g. in S3) back up and restore **independently**, so a
|
|
@@ -2307,20 +2547,6 @@ declare function dropSegment(ref: SegmentRef, deps: DropDeps, options: {
|
|
|
2307
2547
|
* to confirm a reported tear.
|
|
2308
2548
|
*/
|
|
2309
2549
|
|
|
2310
|
-
/** Default in-flight fan-out for the consistency scan — bounded, no thundering herd. */
|
|
2311
|
-
/**
|
|
2312
|
-
* Default ceiling on how many registry records one consistency check may hold resident: 250,000.
|
|
2313
|
-
*
|
|
2314
|
-
* **Named `maxScanSegments`, not `maxSegments`, on purpose.** Compaction already has a `maxSegments`
|
|
2315
|
-
* (`CR_COMPACT_MAX_SEGMENTS`) that caps how much work one cycle does and then *continues normally*. This one
|
|
2316
|
-
* caps how much a scan may hold and *refuses* past it. Same-sounding names with opposite behaviour at the
|
|
2317
|
-
* limit is a trap; pairing this with `maxWarmScanBytes` instead makes both read as what they are — ceilings on
|
|
2318
|
-
* a scan that fail closed.
|
|
2319
|
-
*
|
|
2320
|
-
* Generous — the compaction docs target 100K+ segment fleets — while still bounding a DR drill's memory to
|
|
2321
|
-
* something a modest operator box survives. Raisable, because a ceiling you cannot lift is a landmine.
|
|
2322
|
-
*/
|
|
2323
|
-
declare const DEFAULT_MAX_SCAN_SEGMENTS = 250000;
|
|
2324
2550
|
interface ConsistencyIssue {
|
|
2325
2551
|
readonly segment: string;
|
|
2326
2552
|
readonly namespace?: string;
|
|
@@ -2532,4 +2758,4 @@ declare class CrbmWriter {
|
|
|
2532
2758
|
private buildFooter;
|
|
2533
2759
|
}
|
|
2534
2760
|
|
|
2535
|
-
export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type DropDeps, type DropResult, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, dropSegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
|
|
2761
|
+
export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETIRE_LIMIT, DEFAULT_RETRY_POLICY, DEFAULT_TOMBSTONE_GRACE_MS, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type DropDeps, type DropResult, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, GovernanceMeta, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MIN_EXPIRES_AT_MS, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetentionDeps, type RetentionPolicy, type RetireEntry, type RetireExpiredOptions, type RetireExpiredResult, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, type SetRetentionResult, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, clearSegmentRetention, collectWithinBudget, compactSegment, destroySegment, drainRegistry, dropSegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, getSegmentRetention, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, readRetentionPolicy, resolveBudget, resolvePerOpBudget, retireExpired, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, setSegmentRetention, splitId, validateCompactionOptions, validateMaxScanSegments, validateSegmentRef, withRetry, writeCrbmGeneration };
|