@cloudbitmaps/core 0.9.0 → 0.10.0

package/dist/core/ports.d.ts

@@ -0,0 +1,340 @@
+/**
+ * Storage-driver contracts the engine depends on.
+ *
+ * Two tiers and one pointer. **Storage** is immutable object storage holding `.crbm` generations (`IStorageDriver`,
+ * read by the engine through the per-chunk `StorageChunkSource` view the `.crbm` reader implements); the
+ * **registry** is the authoritative per-segment record — which generation is current, the wrapped DEKs, the
+ * governance metadata — under optimistic concurrency (`IRegistryDriver`). Drivers move opaque bytes and OCC
+ * tokens; they never understand roaring or the `.crbm` layout.
+ */
+import type { BlobSink } from './blob.js';
+import type { WrappedDek } from './crypto.js';
+/** Opaque optimistic-concurrency token — unique per write, compared by equality only. */
+export type Token = string;
+export interface SegmentRef {
+    readonly namespace?: string;
+    readonly segment: string;
+}
+export interface ChunkRef extends SegmentRef {
+    readonly chunkKey: number;
+}
+/** Identifies one immutable `.crbm` object — a single generation of a segment. */
+export interface GenKey extends SegmentRef {
+    readonly generation: number;
+}
+/**
+ * A segment's grounded on-disk footprint — the current generation's Storage object bytes, read cheaply from the
+ * `.crbm` footer/index (no payload reads). Powers the grounded `costReport()`.
+ */
+export interface SegmentSize {
+    readonly sizeBytes: number;
+}
+/** Per-chunk read view of the immutable Storage tier (implemented by the `.crbm` reader). */
+export interface StorageChunkSource {
+    /** Read-only bytes for the chunk, or `null` if absent. Callers must not mutate the buffer. */
+    getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
+    listChunkKeys(ref: SegmentRef): Promise<number[]>;
+    /**
+     * Optional: the current generation's grounded size, cheaply (from the already-parsed `.crbm` index — no
+     * payload reads), or `null` if the segment has no Storage generation. Powers the grounded `costReport()`.
+     */
+    sizeOf?(ref: SegmentRef): Promise<SegmentSize | null>;
+    /**
+     * Optional: the current generation's **per-chunk cardinality** (`chunkKey → count`), read from the
+     * already-parsed `.crbm` index with **no payload reads**, or `null` if the segment has no Storage generation.
+     * Powers the free `count()` — the engine sums the index instead of fetching a single chunk. A source with no
+     * index (e.g. the in-memory source) omits this, and `count()` falls back to fetching every chunk.
+     */
+    cardinalities?(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
+    /**
+     * Optional: the segment's **current generation number** as this source resolves it right now (registry
+     * `currentGen`, or the highest storage generation), or `null` if the segment has no Storage generation. The engine
+     * keys its chunk cache by this so a generation bump (a load's publish) is observed — a new generation
+     * misses the cache instead of serving a stale decoded chunk, and an erased id can't resurrect from a cached
+     * superseded chunk. Cheap: served from the source's own (short-TTL-refreshed) snapshot, **not** a fresh
+     * backend read per call. A source that pins one immutable generation for its whole lifetime and never
+     * refreshes may omit this — the engine then keys the cache without a generation.
+     */
+    currentGeneration?(ref: SegmentRef): Promise<number | null>;
+    /**
+     * Optional: forget everything this source has derived from `ref` — its resolved snapshot, its open reader,
+     * and any key material that reader captured — so the next read resolves from the registry again.
+     *
+     * The TTL refresh and the generation-keyed chunk cache are both built for **a publish that advances
+     * `currentGen`**. Neither covers an event that *destroys* what the cache was derived from: an erasure that
+     * deletes the generation holding the bit, a `dropSegment`, a crypto-shred, a retirement. After one of those
+     * a source that had already resolved the segment keeps answering from memory — with no backend read at all,
+     * so no storage-side control can close the window — until its TTL lapses, and **never** if it has no clock
+     * or `cache.genTtlMs: 0` ("pin forever").
+     *
+     * Callers that destroy or retire a segment must call this. It is synchronous and best-effort: dropping
+     * memoized state cannot fail, and a source that memoizes nothing may omit the method entirely.
+     */
+    invalidate?(ref: SegmentRef): void;
+    /**
+     * Optional: does this segment **exist as a registered name**, as distinct from resolving to no generation?
+     *
+     * `currentGeneration` collapses two very different states into `null`: a name nobody ever created, and a
+     * segment that exists and is legitimately empty. For a *read* that distinction does not matter — both answer
+     * empty. For an **operand of a combine** it is the difference between a suppression list with nobody on it
+     * yet and a suppression list you misspelled, and those must not look alike: the first correctly suppresses
+     * nothing, the second silently suppresses nothing while you believe it is working.
+     *
+     * Consulted **only** when an operand resolved to no chunks at all, so a normal combine never calls it.
+     */
+    exists?(ref: SegmentRef): Promise<boolean>;
+    /**
+     * Optional: an opaque token identifying **which bytes** a read of this segment will see right now —
+     * the generation *and* the incarnation of the name it belongs to.
+     *
+     * {@link StorageChunkSource.currentGeneration} is not enough to key a decoded-chunk cache, and the gap is not
+     * theoretical: `nextGeneration` returns `max(currentGen, highest object) + 1`, so it **restarts at 0** once
+     * a registry row is purged and the bucket emptied. A retired, re-created name therefore serves different
+     * data at the same `currentGen`, and a cache keyed on `(segment, chunk, generation)` hands back the previous
+     * incarnation's ids — an erased id reappearing, with no read to intercept.
+     *
+     * Note that putting the incarnation in the *object key* would not fix this: the new incarnation still starts
+     * at generation 0, so the cache key collides either way. The identity has to reach the cache.
+     *
+     * Returns `null` when the segment resolves to no generation. A source that pins one immutable generation for
+     * its lifetime may omit this, and the engine falls back to the generation alone.
+     */
+    currentVersion?(ref: SegmentRef): Promise<string | null>;
+}
+/** Capabilities a Storage driver advertises; validated at wiring time, fail-fast. */
+export interface StorageCaps {
+    /** REQUIRED — the format relies on byte-range reads. */
+    readonly rangeRead: true;
+    /** Largest single object the backend accepts (informs single-object-vs-shard — future). */
+    readonly maxObjectBytes: number;
+    /** Optional: enables the pure-object `LATEST`-pointer registry variant (S3 conditional put). */
+    readonly conditionalPut?: boolean;
+}
+/**
+ * Immutable object storage for `.crbm` generations. A "dumb byte mover": it
+ * understands neither roaring nor the `.crbm` layout, only opaque bytes addressed by a {@link GenKey}.
+ * The core never reuses a key, so puts are write-once.
+ */
+export interface IStorageDriver {
+    capabilities(): StorageCaps;
+    /**
+     * Stream a new immutable generation. The driver opens a destination, hands `write` a {@link BlobSink},
+     * then atomically commits (and computes the content hash). Throws if the key already exists (write-once).
+     */
+    putImmutable(key: GenKey, write: (sink: BlobSink) => Promise<void>): Promise<{
+        size: number;
+        sha256: string;
+    }>;
+    /** Range read; the caller bounds-checks. Out-of-range is rejected, never a short/adjacent read. */
+    getRange(key: GenKey, offset: number, length: number): Promise<Uint8Array>;
+    /** Speculative tail read: the last `min(maxBytes, size)` bytes + the total object size. */
+    getTail(key: GenKey, maxBytes: number): Promise<{
+        bytes: Uint8Array;
+        size: number;
+    }>;
+    delete(key: GenKey): Promise<void>;
+    /** Enumerate the generations present for a segment (orphan sweep / latest-gen resolution). */
+    list(ref: SegmentRef): AsyncIterable<GenKey>;
+}
+/**
+ * Lifecycle status of a segment. `active` is the steady state; `destroyed` is the post-crypto-shred tombstone
+ * (the row is kept for audit but the segment is logically gone). `compacting`/`erasing` are **reserved** —
+ * stored and round-tripped so a row written by an earlier build still reads, but no writer in this build sets
+ * them.
+ */
+export type RegistryStatus = 'active' | 'compacting' | 'erasing' | 'destroyed';
+/**
+ * Free-form, JSON-serializable governance metadata — a **plain object** (both registry boundaries reject `null`,
+ * an array or a primitive, because a reader that tests `'key' in meta` would throw an untyped `TypeError`).
+ *
+ * `retention` has one **reserved key with defined semantics**: `expiresAt`, an absolute epoch-ms after which a
+ * retention sweep may retire the segment — written by `setSegmentRetention`, parsed by `readRetentionPolicy`. Any
+ * other key is yours (a legal hold, a note, an owner) and is preserved across policy writes. `residency` is still
+ * shape-later: stored and round-tripped, no semantics attached.
+ */
+export type GovernanceMeta = Record<string, unknown>;
+/**
+ * One registry row — the authoritative per-segment record. Exactly one per segment.
+ */
+export interface RegistryRecord extends SegmentRef {
+    /**
+     * **The** authoritative LATEST pointer: which immutable Storage generation is current — or **`null` for a segment
+     * that has no Storage generation yet.**
+     *
+     * `null` is not "unknown", it is a positive statement: *this segment exists and has no Storage data.* It is what
+     * lets a segment have a registry row **before its first load** — `setSegmentRetention` mints one so a policy
+     * can be recorded ahead of the data, and so the segment is reachable by `registry.list()` and therefore by
+     * retention sweeps, `checkConsistency` and every other fleet-wide operation. The alternative — a row with
+     * `currentGen: 0` and no object behind it — is the forbidden `missing-storage-generation` state, which fails per
+     * read with `NotFoundError`. Generation resolution maps `null` onto the same path a segment with **no row**
+     * takes: every read answers empty, and the first publish advances the pointer.
+     */
+    readonly currentGen: number | null;
+    /**
+     * Per-segment data-key (DEK) wrappings for encryption-at-rest: the DEK envelope-wrapped under one or more KEKs
+     * (active + optional recovery). Reading unwraps with any held KEK; **crypto-shred deletes this whole list**,
+     * making the segment's at-rest bytes permanently unrecoverable. Absent ⇒ the segment is cleartext.
+     */
+    readonly wrappedDeks?: readonly WrappedDek[];
+    /**
+     * Optional **external** key reference (reserved) — e.g. a KMS key ARN for a future KMS keystore adapter that
+     * keeps wrapped material in the KMS rather than in-band {@link wrappedDeks}. Unused by the in-process BYOK
+     * keystore. Clearable on crypto-shred.
+     */
+    readonly keyId?: string;
+    readonly status: RegistryStatus;
+    /**
+     * Governance policy. `retention.expiresAt` drives the retention sweep (see {@link GovernanceMeta}); `residency`
+     * is stored and round-tripped with no semantics yet. Both must be plain objects, and both must survive a
+     * `list()` projection — a fleet sweep reads the policy from the enumeration rather than per-segment.
+     */
+    readonly retention?: GovernanceMeta;
+    readonly residency?: GovernanceMeta;
+    /** Epoch-ms of creation / last mutation (from the driver's injected clock). */
+    readonly createdAt: number;
+    readonly updatedAt: number;
+    /** Opaque OCC token — compare-by-equality, never reused (ABA-safe). */
+    readonly token: Token;
+}
+/** The caller-settable fields at {@link IRegistryDriver.create} (audit + token are driver-managed). */
+export interface NewRegistryRecord {
+    /** `null` ⇒ the segment has no Storage generation yet — see {@link RegistryRecord.currentGen}. */
+    readonly currentGen: number | null;
+    readonly wrappedDeks?: readonly WrappedDek[];
+    readonly keyId?: string;
+    /** Defaults to `'active'`. */
+    readonly status?: RegistryStatus;
+    readonly retention?: GovernanceMeta;
+    readonly residency?: GovernanceMeta;
+}
+/** Fields a {@link IRegistryDriver.compareAndSwap} may mutate (identity + audit + token are off-limits). */
+export type RegistryPatch = Partial<Pick<RegistryRecord, 'currentGen' | 'wrappedDeks' | 'keyId' | 'status' | 'retention' | 'residency'>>;
+/** Capabilities a registry driver advertises; validated fail-fast at wiring time. */
+export interface RegCaps {
+    /** REQUIRED — `currentGen` feeds read correctness + the publish CAS, so reads must be strongly consistent. */
+    readonly strongRead: true;
+}
+/**
+ * Brand for {@link StorageBackend}.
+ *
+ * `Symbol.for` rather than a class check, for the same reason the error brands are: a class object is only
+ * shared inside one package's bundle. `@cloudbitmaps/roaring` carries its own copy of core, so a backend
+ * built from one package and checked by the other would fail an `instanceof` on an ordinary install — no
+ * version skew required — as would a genuine second copy from a skew or a duplicating bundler. A registered
+ * symbol is identity-stable across all of them, and across realms.
+ *
+ * The string says `cloudbitmaps` while the error brands still say `cloud-roaring`: those are locked, because
+ * changing one would break error identity against already-published bundles. A new brand has no such
+ * constraint, so it uses the current name.
+ */
+declare const STORAGE_BACKEND_BRAND: unique symbol;
+/**
+ * Where a store keeps everything: the generations, and the pointer saying which one is current.
+ *
+ * **Branded on purpose.** The two fields alone are not enough to qualify, and that is the whole point. This
+ * shape is also {@link LoadDeps}/{@link GenerationDeps}, so before the brand any `{ storage, registry }`
+ * object literal satisfied it — including one assembling halves from two *unrelated* stores, which the store
+ * accepted and then answered **empty** for a segment that holds data, because it read a pointer from a place
+ * nothing had ever been written. That is the exact silent-empty failure one-class-per-backend exists to
+ * remove, and it was reachable in five lines.
+ *
+ * So a backend comes from `MemoryStorage`, `LocalFsStorage`, `S3Storage`, `GcsStorage` or
+ * `AzureBlobStorage` — each deriving both halves from one bucket and one prefix — or from
+ * {@link createBackend}, which is the same thing said on purpose. The halves stay readable as `.storage` /
+ * `.registry`, because the free functions genuinely need them.
+ *
+ * **What this stops is the accident, not a determined caller.** The brand is a registered symbol, so it can
+ * be written by hand; nothing here is a security boundary. It stops the two spellings people reach for
+ * without deciding to — an object literal, and a spread of a real backend.
+ */
+export interface StorageBackend {
+    /** Cross-bundle brand. Set by the backend classes; not part of the public surface. */
+    readonly [STORAGE_BACKEND_BRAND]: true;
+    /** Where the immutable `.crbm` generations live. */
+    readonly storage: IStorageDriver;
+    /** Where the `currentGen` pointer, the discovery index and the wrapped DEKs live. */
+    readonly registry: IRegistryDriver;
+}
+/**
+ * Build a backend from two halves you supply yourself.
+ *
+ * **Reach for a backend class first** — `MemoryStorage`, `LocalFsStorage`, `S3Storage`, `GcsStorage`,
+ * `AzureBlobStorage`. Each derives both halves from one bucket and one prefix, so they cannot disagree, and
+ * that is the whole reason the classes exist.
+ *
+ * This is the door for the cases a class cannot express, and they are real: a driver wrapped for auditing,
+ * metrics, tenant scoping or client-side encryption; a registry in a database you already run; a
+ * fault-injecting double in a test. What it is NOT is a shortcut — naming it is the point. The plain
+ * `{ storage, registry }` literal is refused precisely because it let the two halves come from unrelated
+ * places by accident, and answer **empty** rather than fail.
+ *
+ * It cannot check that the halves agree: `IStorageDriver` and `IRegistryDriver` do not expose a location, so
+ * nothing here can compare one. Calling this is you taking that on.
+ */
+export declare function createBackend(halves: {
+    readonly storage: IStorageDriver;
+    readonly registry: IRegistryDriver;
+}): StorageBackend;
+/**
+ * Stamp the brand on a backend, **non-enumerably**.
+ *
+ * Non-enumerable is load-bearing, not tidiness. A class field (`readonly [BRAND] = true`) emits an
+ * *enumerable* own property, and spread and `Object.assign` copy exactly those — so
+ * `{ ...backend, registry: other.registry }` carried the brand and was accepted, reproducing the silent-empty
+ * bug this whole thing exists to close. That spelling is also the most idiomatic way a JS developer swaps one
+ * half, which means the audience `createBackend` was added for walked straight past it.
+ *
+ * This does not make the brand unforgeable — the symbol is registered, so anyone who wants it can write
+ * `Symbol.for('cloudbitmaps.storage-backend')`. That is deliberate effort equivalent to calling
+ * `createBackend`, and it is not what the check is for. The check is for the accident.
+ */
+/**
+ * The brand key, for the type-only `declare` field a backend class carries.
+ *
+ * Exported from `@cloudbitmaps/core/driver-kit` and part of the driver contract: a third-party storage
+ * package needs it to declare the field that {@link brandAsBackend} then stamps at runtime.
+ */
+export declare const STORAGE_BACKEND: typeof STORAGE_BACKEND_BRAND;
+export declare function brandAsBackend<T extends {
+    storage: IStorageDriver;
+    registry: IRegistryDriver;
+}>(target: T): T & StorageBackend;
+/**
+ * Is this a branded backend — from a backend class or from {@link createBackend}?
+ *
+ * Checks the brand, not the shape — see {@link StorageBackend} for why the shape alone is not enough.
+ */
+export declare function isStorageBackend(value: unknown): value is StorageBackend;
+/**
+ * Per-segment registry — the authoritative source of `currentGen`, the discovery index, and the wrapped-DEK
+ * holder. One row per segment under OCC: a never-reused, equality-compared {@link Token} (ABA-safe across
+ * delete→recreate).
+ */
+export interface IRegistryDriver {
+    capabilities(): RegCaps;
+    /** The segment's record, or `null` if it doesn't exist (or was deleted). */
+    get(ref: SegmentRef): Promise<RegistryRecord | null>;
+    /** Create the row; throws {@link WriteConflictError} if it already exists (use CAS to mutate). */
+    create(ref: SegmentRef, record: NewRegistryRecord): Promise<{
+        token: Token;
+    }>;
+    /** Server-side compare-and-set: apply `patch` iff the stored token equals `expected`, else `WriteConflictError`. */
+    compareAndSwap(ref: SegmentRef, expected: Token, patch: RegistryPatch): Promise<{
+        token: Token;
+    }>;
+    /**
+     * Discovery: every **existing** record, optionally scoped to one namespace. Order is unspecified.
+     *
+     * "Existing" means not `delete`d. A **`destroyed` tombstone is still a record and must be yielded** — a driver
+     * that filters by `status` breaks callers silently, and two already depend on seeing them: `runConsistencyCheck`
+     * skips them itself, and the retention sweep can only clean up a tombstone row it can see (filtering it makes
+     * the cleanup a permanent no-op, indistinguishable from having nothing to do, while dead rows accumulate). Same
+     * rule for every other field: a row with **`currentGen: null`** must be yielded like any other, and `retention`
+     * must survive the projection — a fleet sweep reads the policy straight out of this enumeration rather than
+     * paying a `get()` per segment, so a `list()` that drops the field means nothing ever expires, silently.
+     */
+    list(namespace?: string): AsyncIterable<RegistryRecord>;
+    /** Remove the row (tombstoned for ABA-safety — a later `create` still gets a fresh, greater token). */
+    delete(ref: SegmentRef): Promise<void>;
+}
+export {};

package/dist/core/registry-scan.d.ts

@@ -0,0 +1,38 @@
+import type { IRegistryDriver, RegistryRecord } from './ports.js';
+/**
+ * Ceiling on registry records one fleet-wide scan holds resident. Raise it via the caller's `maxScanSegments` when
+ * the fleet really is that large *and* the memory is there; narrow the scan with a `namespace` otherwise.
+ */
+export declare const DEFAULT_MAX_SCAN_SEGMENTS = 250000;
+/** Fail fast on a bad ceiling BEFORE the (possibly huge) scan, not after. */
+export declare function validateMaxScanSegments(value: number, op: string): void;
+/**
+ * Drain every registry record (optionally one namespace) into an array, refusing past `maxScanSegments`.
+ *
+ * The bound is checked **before** each push, so the array never exceeds the ceiling the caller agreed to — the
+ * earlier form pushed first and then threw, which meant the very row that broke the budget was already resident,
+ * and a caller sizing a container against `maxScanSegments` was off by one row at the worst possible moment.
+ */
+export declare function drainRegistry(registry: IRegistryDriver, options: {
+    namespace?: string;
+    maxScanSegments: number;
+    op: string;
+}): Promise<RegistryRecord[]>;
+/**
+ * Is this record **bookkeeping** rather than a segment? Covers every reserved family — today the due-index
+ * pointers. **Every unscoped fleet-wide enumeration must skip these**: the bounded drain,
+ * the export/eject scan, and the all-namespaces GDPR paths.
+ *
+ * This is the ONE place a reserved family is declared, and a new one belongs here rather than at the call
+ * sites. An earlier cut inlined the comparison per site and shipped with three missed — including
+ * `subjectReport`, where the rows consumed an Art. 15 request's per-op budget. A filter you have to remember at
+ * each site is a check that cannot fire.
+ */
+export declare function isReservedRow(record: Pick<RegistryRecord, 'namespace'>): boolean;
+/**
+ * Filter bookkeeping rows out of a registry listing **before** anything downstream pays for them. Needed where
+ * the enumeration is consumed inside a budgeted collector rather than a plain loop: charging a GDPR Art. 15
+ * request's per-op budget for pointer rows can refuse a subject report for a reason that has nothing to do with
+ * the subject.
+ */
+export declare function excludingReservedRows(source: AsyncIterable<RegistryRecord>): AsyncIterable<RegistryRecord>;

package/dist/core/retention-sweep.d.ts

@@ -0,0 +1,190 @@
+/**
+ * The retention **sweep** — the thing that acts on the policies `setSegmentRetention` records.
+ *
+ * `retireExpired` enumerates the registry, selects the segments whose `expiresAt` has passed, and retires each
+ * one through {@link dropSegment}. It deliberately **delegates rather than reimplements**: the registry → Storage
+ * ordering, the re-sweep for an object a load was still writing, and the `generationsRemaining` report are all
+ * load-bearing and already live there. A sweep that open-coded the deletions would be a second
+ * implementation of the most dangerous ordering in the library.
+ *
+ * **This is a call, not a daemon.** Nothing here schedules itself. You run it from whatever heartbeat your
+ * deployment already has — an EventBridge rule, a Kubernetes CronJob, a queue consumer, the job that runs your
+ * loads — and the library stays a library: it has to behave identically in a Lambda, an edge isolate and a
+ * long-lived server, and a timer that only works in one of those is worse than none.
+ *
+ * Three properties make it safe to point at a fleet:
+ *
+ *  - **`dryRun` at the sweep level.** `dropSegment`'s `confirmSegment` guard is vacuous in a loop (it is the same
+ *    variable twice), so the meaningful preview is here: it reports what it *would* retire, per segment, and
+ *    touches nothing.
+ *  - **A per-cycle `limit`.** Clock skew, a bad backfill, or a namespace-wide policy mistake should cost one
+ *    bounded batch, not the fleet. When the limit bites, the result says so (`limited: true`) rather than looking
+ *    like a completed sweep.
+ *  - **A ledger, never an exception.** A per-segment fault is recorded and the sweep continues — the caller's
+ *    load-bearing question is *which* segments were retired, and a throw from the middle of the loop answers it
+ *    for none of them while having already retired some. Same posture as `eraseNamespace`.
+ *
+ * It also **purges the tombstones its own retirements leave behind**. A retired segment that had a registry row
+ * gets a `destroyed` row, and one dead row per retired daily bucket — or per retired dedup wave — is exactly the
+ * registry litter `dropSegment` already refuses to create for a row-less accumulator. Purging is narrow on
+ * purpose: only a tombstone that still carries an **expired retention policy** (so it is attributably ours, never
+ * a GDPR crypto-shred), only after a grace period, and only once Storage is provably empty for it —
+ * because deleting the row is what makes the name reusable and takes the segment out of reach of
+ * `gcOrphanGenerations`.
+ */
+import { type IAuditSink } from './audit.js';
+import type { DropDeps, DropResult } from './erasure.js';
+/** Default cap on retirements per sweep — a bounded batch, so a policy mistake costs one batch, not the fleet. */
+export declare const DEFAULT_RETIRE_LIMIT = 100;
+/**
+ * Default delay before a retirement's own tombstone row is purged: 24 h.
+ *
+ * The row is a fence — while it exists, `publishGeneration` and `bulkLoadCrbmGeneration` refuse the segment, so
+ * a load that was mid-write when the drop landed cannot resurrect it. That window is
+ * seconds to minutes in practice; a day of margin costs one tiny row and removes any need to reason about it.
+ */
+export declare const DEFAULT_TOMBSTONE_GRACE_MS = 86400000;
+export interface RetireExpiredOptions {
+    /** Scope the sweep to one namespace. Omit to sweep every namespace the registry knows. */
+    readonly namespace?: string;
+    /**
+     * The instant to compare policies against (epoch-ms). Required here because `core/` takes its time from the
+     * caller, never from the platform — `store.retireExpired()` fills it in from the store's clock.
+     */
+    readonly now: number;
+    /** Maximum segments to retire in this cycle (default 100). */
+    readonly limit?: number;
+    /** Report what would be retired and change nothing. */
+    readonly dryRun?: boolean;
+    /** Forwarded to each `dropSegment`, so every retirement lands in the audit trail as `segment.dispose`. */
+    readonly audit?: IAuditSink;
+    /** Ceiling on rows enumerated (default 250,000); exceeding it throws. */
+    readonly maxScanSegments?: number;
+    /**
+     * **Where the candidates come from.**
+     *
+     * - `'fleet'` (default) — drain `registry.list()` and filter. Cost tracks the **fleet**, every cycle, even
+     *   when nothing expires. Complete by construction: it cannot miss a policy.
+     * - `'index'` — read only the due buckets of the due index. Cost tracks **what is
+     *   expiring**. Each candidate's live row is still re-read before anything is decided, so a stale pointer
+     *   costs one read and retires nothing.
+     *
+     * **`'index'` is not a drop-in replacement for `'fleet'`; it is the fast half of a pair.** A policy written
+     * before the index existed, or one whose pointer write failed (`indexed: false`), has no pointer — so a
+     * deployment that *only* ever runs `'index'` will never retire those. Run `'fleet'` periodically as the
+     * repair pass. The default stays `'fleet'` so that upgrading changes nothing about what gets retired.
+     */
+    readonly scan?: 'fleet' | 'index';
+    /**
+     * **The shards this worker owns**, with {@link totalShards}. Without them every replica sweeps the whole
+     * fleet and contends over the same segments. A stable hash of the segment key, so a worker owns the same slice
+     * across restarts.
+     */
+    readonly shards?: readonly number[];
+    /** Total shards the fleet is split into. Required with {@link shards}; ignored without it. */
+    readonly totalShards?: number;
+    /**
+     * How many **past** buckets an `'index'` scan reads besides the current one (default 7, one per day).
+     * A sweep that did not run — scaled to zero, a failed deploy, a paused
+     * schedule — leaves its buckets behind, and this is how far back a later cycle reaches for them. Bounded so a
+     * long outage costs a bounded number of list calls per cycle rather than one per day since the epoch;
+     * anything older is the `'fleet'` repair pass's job.
+     */
+    readonly lookbackBuckets?: number;
+    /**
+     * Whether to delete the tombstone rows this sweep's own past retirements left (default `true`). Set `false` to
+     * keep every tombstone forever — the right choice if something outside this library treats the presence of a
+     * `destroyed` row as an attestation.
+     *
+     * Two knobs rather than one `number | 'never'`, deliberately: `0` would have had to mean "purge immediately"
+     * here while `cache.genTtlMs: 0` in this same library means "pin forever", and one option whose zero is the
+     * opposite of another's is a reading hazard for whoever tunes both.
+     */
+    readonly purgeTombstones?: boolean;
+    /** How long a retirement's tombstone must age before this sweep deletes it (default 24 h). */
+    readonly tombstoneGraceMs?: number;
+}
+/** What happened to one segment in a sweep. */
+export type RetireEntry = {
+    readonly segment: string;
+    readonly namespace?: string;
+    /** Retired: `result` is `dropSegment`'s full report — **check `generationsRemaining`**. */
+    readonly action: 'retired';
+    readonly expiresAt: number;
+    readonly result: DropResult;
+    /**
+     * Present when the retirement **completed the destructive part and then faulted** — the tombstone is written
+     * and the segment reads empty, but something after that (the Storage sweep) threw. The storage may not be fully
+     * reclaimed; re-run. Reported as `retired` rather than `skipped` because the segment really is retired, and
+     * saying otherwise is the one thing a caller cannot recover from.
+     */
+    readonly fault?: `failed: ${string}`;
+} | {
+    readonly segment: string;
+    readonly namespace?: string;
+    /** `dryRun` only — what a real sweep would have retired, with `dropSegment`'s own preview attached. */
+    readonly action: 'would-retire';
+    readonly expiresAt: number;
+    readonly result: DropResult;
+} | {
+    readonly segment: string;
+    readonly namespace?: string;
+    /** A tombstone row from an earlier retirement was deleted (or would be, under `dryRun`). */
+    readonly action: 'purged-tombstone' | 'would-purge-tombstone';
+} | {
+    readonly segment: string;
+    readonly namespace?: string;
+    readonly action: 'skipped';
+    /**
+     * `'invalid-policy'` — the row has an `expiresAt` that is not usable (hand-edited, or restored from another
+     * schema). Reported rather than ignored: reading as "never expires" on a segment someone believes is
+     * expiring is the silence that costs a retention commitment.
+     * `'limit'` — eligible, but this cycle's `limit` was already spent. Re-run to continue.
+     * `'tombstone-not-empty'` — a tombstone whose Storage generations are not gone even after a GC
+     * attempt, so its row is kept: the row is what keeps the segment reachable by `gcOrphanGenerations` and
+     * refused by every writer. Several causes, all self-healing: the storage really could not be reclaimed,
+     * the collection *declined* because the row changed underneath it (`WriteConflictError`, which this
+     * sweep swallows deliberately), or this was a `dryRun`, which reports the reason without attempting the
+     * collection at all. Except under `dryRun` the next cycle retries.
+     * `'policy-changed'` — the live row no longer says "expired" (a `clearRetention`, a new `expiresAt`, or
+     * someone else's drop landed between the enumeration and this segment's turn). Not an error: the sweep
+     * re-reads before every deletion precisely so cancelling an expiry works on a sweep already in flight.
+     * `` `failed: …` `` — any fault, isolated to this segment. A fault that happened *after* the tombstone
+     * landed is reported as `retired` with a `fault` instead, because that segment IS retired.
+     */
+    readonly reason: 'invalid-policy' | 'limit' | 'tombstone-not-empty' | 'policy-changed' | `failed: ${string}`;
+};
+export interface RetireExpiredResult {
+    /** Registry rows enumerated. */
+    readonly scanned: number;
+    /** Rows whose policy said "expired" — including any the `limit` deferred. */
+    readonly eligible: number;
+    /**
+     * Segments **actually retired**. Zero under `dryRun` — see `wouldRetire`. Kept honest because this is the field
+     * most likely to end up on a dashboard, and the CLI emits it: a counter that means "deleted" in one mode and
+     * "would delete" in another produces phantom deletions on any graph that does not also join on `dryRun`.
+     */
+    readonly retired: number;
+    /** Segments a real sweep **would** have retired. Only ever non-zero under `dryRun`. */
+    readonly wouldRetire: number;
+    /** Tombstone rows actually deleted. Zero under `dryRun`; the `would-purge-tombstone` entries carry the preview. */
+    readonly tombstonesPurged: number;
+    /** True when `limit` cut the cycle short — **more segments are still eligible**. Re-run. */
+    readonly limited: boolean;
+    readonly dryRun: boolean;
+    /** Per-segment ledger. Inspect it: a `skipped` or failed entry is a segment that still holds data. */
+    readonly entries: readonly RetireEntry[];
+}
+/**
+ * Retire every segment whose retention policy has expired, and clean up the tombstones earlier sweeps left.
+ *
+ * Returns a ledger; **never throws for a per-segment fault** (those become entries). It does throw for a bad
+ * argument, and for a fleet larger than `maxScanSegments` — a scan that cannot be held in memory is a fail-loud
+ * condition, not a partial result to be mistaken for a complete sweep.
+ */
+/**
+ * How many past buckets an index scan reads besides the current one. A week: long enough that a weekend outage
+ * or a paused schedule recovers on its own, short enough that a cycle is eight list calls rather than hundreds.
+ */
+export declare const DEFAULT_LOOKBACK_BUCKETS = 7;
+export declare function retireExpired(deps: DropDeps, options: RetireExpiredOptions): Promise<RetireExpiredResult>;

package/dist/core/retention.d.ts

@@ -0,0 +1,67 @@
+import type { GovernanceMeta, IRegistryDriver, SegmentRef } from './ports.js';
+/**
+ * The smallest accepted `expiresAt`: 2001-09-09, the point below which an epoch-**seconds** value would land.
+ *
+ * This is a deliberate guard against the single most damaging typo available here. `Date.now() / 1000 + 30 * 86400`
+ * is a natural thing to write, and it produces a number ~1.8 billion — an instant in **1970**, which is to say
+ * *already expired*. Without this check, that typo is not a validation error, it is a sweep that retires the
+ * segment on its very next pass. Anything below this floor is far likelier to be seconds than a genuine intent to
+ * expire a segment two decades ago, and the caller who really means "retire it now" can say so with `dropSegment`.
+ */
+export declare const MIN_EXPIRES_AT_MS = 1000000000000;
+/** A segment's retention policy: the instant it becomes eligible for retirement. */
+export interface RetentionPolicy {
+    /**
+     * Absolute epoch-**milliseconds**. Once `now >= expiresAt`, a retention sweep may retire the segment — tombstone
+     * it and delete its Storage generations. A value in the past is legal (backfilling a policy onto existing
+     * buckets is a normal thing to do) and means "eligible on the next sweep".
+     */
+    readonly expiresAt: number;
+}
+export interface RetentionDeps {
+    readonly registry: IRegistryDriver;
+}
+export interface SetRetentionResult {
+    readonly segment: string;
+    readonly namespace?: string;
+    /** The policy now stored on the row. */
+    readonly expiresAt: number;
+    /**
+     * True iff this call **minted the registry row** — a policy recorded before the segment's first load. The
+     * segment is now enumerable by `registry.list()` and therefore by every fleet-wide operation; the row claims no
+     * Storage generation (`currentGen: null`), so reads answer empty until the first load publishes onto it.
+     */
+    readonly createdRow: boolean;
+    /**
+     * True iff the **due-index pointer** for this policy was written, so a fast sweep will find this segment by
+     * reading only its expiry day instead of scanning the fleet.
+     *
+     * `false` is a **degradation, not an error**, and the policy is committed either way: either the ref is too
+     * long to encode into one index-row name, or the pointer write failed. The full-scan repair pass still sees
+     * the segment's own row, so it is retired on the repair cadence rather than the fast one — slower, never
+     * never. Alarm on a *sustained* run of `false`, not on one.
+     */
+    readonly indexed: boolean;
+}
+/**
+ * Read the policy off a registry row's `retention` metadata. Returns `null` when there is no policy, and
+ * `'invalid'` when the key is present but unusable (wrong type, or below the epoch-seconds floor).
+ *
+ * The three-way answer exists so a fleet sweep can *report* a malformed policy instead of choosing between two
+ * bad options: treating it as "no policy" hides an operator error on a segment someone believes is expiring, and
+ * throwing aborts a ledger over one bad row. Nothing writes an invalid policy through
+ * {@link setSegmentRetention} — hand-edited rows, an older/newer writer, or a restore are how it happens.
+ */
+export declare function readRetentionPolicy(meta: GovernanceMeta | undefined): RetentionPolicy | null | 'invalid';
+/** The stored policy for one segment, or `null` if it has none (and `'invalid'` for a malformed one). */
+export declare function getSegmentRetention(ref: SegmentRef, deps: RetentionDeps): Promise<RetentionPolicy | null | 'invalid'>;
+export declare function setSegmentRetention(ref: SegmentRef, deps: RetentionDeps, policy: RetentionPolicy): Promise<SetRetentionResult>;
+/**
+ * Remove a segment's expiry so no sweep will retire it. Returns whether a policy was actually removed (`false`
+ * for a segment with no row, or one that had no policy) — a separate verb from setting one, because "cancel the
+ * expiry" and "expire at time T" are different intents and a magic sentinel value for the first is how a typo
+ * becomes a deletion.
+ *
+ * Other `retention` keys are left in place; if removing the expiry empties the metadata, the field is cleared.
+ */
+export declare function clearSegmentRetention(ref: SegmentRef, deps: RetentionDeps): Promise<boolean>;