@kici-dev/shared 0.0.0 → 0.1.0

package/dist/cold-store/cold-store.d.ts

@@ -0,0 +1,345 @@
+/**
+ * BaseColdStore — the DB-agnostic framework class.
+ *
+ * Multiple concrete subclasses extend this one (e.g., `PlatformColdStore`,
+ * `OrchestratorColdStore`). Each wires the adapter map with its
+ * package-specific adapters.
+ *
+ * Phase B: `runArchiveCycle()` now drives the per-adapter archive loop
+ * (design §4). `fetchRange` / `hasRange` / `countRange` are real — they
+ * resolve manifests via `ListObjectsV2` and GET overlapping chunks
+ * through an in-process LRU.
+ *
+ * ## Advisory-lock namespace
+ *
+ * All Postgres advisory locks acquired by the cold-store framework use
+ * keys derived from `hashtext('cold-store|...')`. The `cold-store|`
+ * prefix is reserved — do NOT reuse it in pipeline or engine code
+ * without coordinating to avoid collisions.
+ *
+ * ## S3 bucket versioning caveat
+ *
+ * When the cold-store bucket has versioning ON (recommended default),
+ * retried chunk PUTs produce new object versions at the same key.
+ * Reconciliation (the `reconcile` CLI) MUST use `ListObjectVersions`,
+ * not `ListObjects`, to detect orphaned objects correctly. The live
+ * read-through uses `ListObjectsV2` because it only cares about the
+ * current version of each manifest.
+ */
+import { type S3Client } from '@aws-sdk/client-s3';
+import { type DbKind } from './key.js';
+import type { ChunkLru } from './lru.js';
+import type { ColdStoreConfig } from './config.js';
+import type { TableAdapter } from './table-adapter.js';
+import type { ArchiveCycleSummary, ChunkManifest } from './types.js';
+export interface ColdStoreFetchRangeArgs<TRow> {
+    db: DbKind;
+    table: string;
+    tenantId: string;
+    fromTs: Date;
+    toTs: Date;
+    decode?: (line: string) => TRow;
+}
+export interface ColdStoreReplayChunkArgs {
+    db: DbKind;
+    table: string;
+    tenantId: string;
+    partitionDate: string;
+    chunkId: string;
+}
+export interface ColdStoreReplayRowArgs {
+    db: DbKind;
+    table: string;
+    tenantId: string;
+    rowId: string | number;
+}
+export interface ColdStoreReplayResult {
+    /** Number of rows newly inserted into PG. */
+    inserted: number;
+    /** Rows whose unique key was already in PG (idempotent re-replay). */
+    skipped: number;
+    /** The chunk that was replayed (`null` when replayRow finds no match). */
+    chunkId: string | null;
+}
+/**
+ * Phase 2 — purge-sweep options.
+ *
+ * `tableFilter` restricts to a single table (used by
+ * `cold-store purge-now <table>`). `bucketFilter` further restricts to a
+ * single bucket subprefix. `limit` caps the number of chunks processed
+ * per call so a single sweep can't OOM the orchestrator on a tenant
+ * with millions of expired chunks. `dryRun=true` lists candidates
+ * without actually deleting anything (the CLI default — operators must
+ * pass `--apply`).
+ */
+export interface PurgeExpiredChunksOpts {
+    tableFilter?: string;
+    bucketFilter?: string;
+    limit?: number;
+    dryRun?: boolean;
+}
+/**
+ * Phase 2 — one row per (chunk attempt, outcome). Returned from
+ * `purgeExpiredChunks` so callers (scheduled job, CLI) can log / report
+ * what happened without re-querying the chunk index.
+ */
+export interface PurgeChunkResult {
+    table: string;
+    tenantId: string;
+    chunkId: string;
+    bucket: string;
+    gzipBytes: number;
+    rowCount: number;
+    outcome: 'purged' | 'dry_run' | 'skipped_locked' | 'failure';
+    /** Populated only when `outcome === 'failure'`. */
+    error?: string;
+}
+export interface PurgeExpiredChunksSummary {
+    results: PurgeChunkResult[];
+    /** Total bytes (gzipped) actually purged from S3. */
+    bytesPurged: number;
+    /** Total chunks actually purged (i.e., outcome === 'purged'). */
+    chunksPurged: number;
+    /** Wall-clock duration of the whole sweep. */
+    durationMs: number;
+}
+/**
+ * Phase 2 — chunk-index row shape returned by
+ * `BaseColdStore.listPurgeableChunks`. Mirrors `cold_store_chunks` row
+ * with a normalized `maxColdDays` field.
+ */
+export interface PurgeableChunk {
+    table: string;
+    tenantId: string;
+    chunkId: string;
+    bucket: string;
+    archivedAt: Date;
+    gzipBytes: number;
+    rowCount: number;
+    maxColdDays: number;
+    objectKey: string;
+}
+/**
+ * Public cold-store API. Phase B implements all four core methods.
+ * Phase F adds `replayChunk` / `replayRow` for the rerun-from-archive
+ * flow.
+ */
+export interface ColdStore {
+    /** Stream archived rows that overlap [fromTs, toTs). */
+    fetchRange<TRow>(args: ColdStoreFetchRangeArgs<TRow>): AsyncIterable<TRow>;
+    hasRange(args: Omit<ColdStoreFetchRangeArgs<unknown>, 'decode'>): Promise<boolean>;
+    countRange(args: Omit<ColdStoreFetchRangeArgs<unknown>, 'decode'>): Promise<number>;
+    /**
+     * Run one archive cycle. By default iterates every registered adapter
+     * in registration order; pass `tableFilter` to restrict to a single
+     * adapter (used by `archive-now <table>` CLI commands so an operator
+     * can flush one table on demand without waiting for the next cron tick).
+     */
+    runArchiveCycle(opts?: {
+        tableFilter?: string;
+    }): Promise<ArchiveCycleSummary>;
+    /**
+     * Phase F — promote every row in a chunk back into PG transactionally.
+     * Idempotent on re-run via the adapter's `ON CONFLICT DO NOTHING`.
+     * Throws if the adapter doesn't implement `replayInsert`, the chunk
+     * is missing, or the contentHash check fails.
+     */
+    replayChunk(args: ColdStoreReplayChunkArgs): Promise<ColdStoreReplayResult>;
+    /**
+     * Phase F — locate the chunk containing `rowId` (via manifest
+     * `minRowId`/`maxRowId` bounds) and replay it. Returns
+     * `chunkId: null` when no manifest matches — caller handles as
+     * "row truly does not exist anywhere".
+     */
+    replayRow(args: ColdStoreReplayRowArgs): Promise<ColdStoreReplayResult>;
+    /**
+     * Phase 2 — purge expired chunks from S3.
+     *
+     * Looks up `cold_store_chunks` rows where
+     * `now() > archived_at + max_cold_days * INTERVAL '1 day'` AND
+     * `max_cold_days != 'forever'`. For each:
+     *   1. Acquire a per-chunk advisory lock so concurrent sweeps can't
+     *      double-delete.
+     *   2. Issue `DeleteObject` for both the data and manifest keys.
+     *   3. Call `adapter.purgeChunkRecord(...)` to transactionally delete
+     *      the row from `cold_store_chunks`, decrement the rollup, and
+     *      write a `purge_chunk` audit row.
+     *
+     * `dryRun=true` (CLI default) skips steps 2–3 and returns the
+     * candidate list without mutating anything. Production scheduled
+     * sweeps pass `dryRun=false`.
+     */
+    purgeExpiredChunks(opts?: PurgeExpiredChunksOpts): Promise<PurgeExpiredChunksSummary>;
+}
+export interface BaseColdStoreDeps {
+    db: DbKind;
+    config: ColdStoreConfig;
+    instanceId: string;
+    /** Caller-supplied logger-like callable; omits pull on shared logger. */
+    log: (level: 'info' | 'warn' | 'error', msg: string, extra?: Record<string, unknown>) => void;
+    /** Caller-supplied LRU; shared across consumers if desired. */
+    chunkCache?: ChunkLru<string, Buffer>;
+    /**
+     * Pre-built S3 client. If omitted, the BaseColdStore constructs one
+     * from `config.storage` via `createS3Client`. Tests pass a mock.
+     */
+    s3Client?: S3Client;
+}
+export declare abstract class BaseColdStore implements ColdStore {
+    protected readonly db: DbKind;
+    protected readonly config: ColdStoreConfig;
+    protected readonly instanceId: string;
+    protected readonly log: BaseColdStoreDeps['log'];
+    protected readonly chunkCache: ChunkLru<string, Buffer> | undefined;
+    protected readonly s3: S3Client;
+    /** Registered table adapters by `table` name. */
+    protected readonly adapters: Map<string, TableAdapter<unknown>>;
+    protected constructor(deps: BaseColdStoreDeps);
+    /**
+     * Register a concrete table adapter. Subclasses typically call this
+     * from their constructor.
+     */
+    protected registerAdapter(adapter: TableAdapter<unknown>): void;
+    /** Test / CLI helper — adapter-map read access. */
+    getAdapter(table: string): TableAdapter<unknown> | undefined;
+    listAdapters(): ReadonlyArray<TableAdapter<unknown>>;
+    /**
+     * CLI-accessible S3 primitives. Exposed so admin CLI commands can run
+     * inspection / reconciliation operations without a direct dependency
+     * on `@aws-sdk/client-s3`. Not intended for hot-path use.
+     */
+    get bucket(): string;
+    get storagePrefix(): string;
+    /** List all object keys under a prefix, paging internally. */
+    listObjectKeys(prefix: string): Promise<string[]>;
+    /** Fetch an object's raw body as a Buffer. */
+    getObjectBody(key: string): Promise<Buffer>;
+    /** Upload a manifest JSON object at `key`. */
+    putManifestObject(key: string, manifest: ChunkManifest): Promise<void>;
+    /**
+     * Delete a single object by key. CLI-accessible companion to
+     * `listObjectKeys` / `getObjectBody`, used by tenant-scoped sweep
+     * paths (e.g. the post-`deleteOrgCascade` S3 cleanup) that operate
+     * on chunks the index already forgot about. Does NOT touch the
+     * `cold_store_chunks` index — callers that need both must follow
+     * the `purgeExpiredChunks` pattern with `withPurgeLock` +
+     * `adapter.purgeChunkRecord`.
+     */
+    deleteObject(key: string): Promise<void>;
+    runArchiveCycle(opts?: {
+        tableFilter?: string;
+    }): Promise<ArchiveCycleSummary>;
+    private processAdapter;
+    /**
+     * Archive one (tenant, partitionDate).
+     *
+     * - Adapters WITHOUT `coldTtlDays`: legacy single-chunk path. Writes one
+     *   chunk per partition at the day-prefix root with a v1 manifest. The
+     *   GC sweep treats v1 chunks as `'forever'`.
+     * - Adapters WITH `coldTtlDays`: Phase 2 per-bucket path. Buffers all
+     *   eligible rows, groups by `coldDaysToBucket(coldTtlDays(row))`,
+     *   then emits one chunk per non-empty bucket under the bucket
+     *   subprefix with a v2 manifest carrying `bucket` + `maxColdDays`.
+     *
+     * In both paths the caller holds the advisory lock for the
+     * partition. Returns the total row count archived across all chunks
+     * so the outer loop can honor `maxRowsPerCycle`.
+     */
+    private archivePartition;
+    /**
+     * Legacy single-chunk-per-partition flow. Adapters that do not
+     * implement `coldTtlDays` get a v1 manifest at the day-prefix root.
+     */
+    private archivePartitionLegacy;
+    /**
+     * Phase 2 per-bucket flow. Buffers all eligible rows, groups by
+     * `coldDaysToBucket(coldTtlDays(row))`, then emits one chunk per
+     * non-empty bucket.
+     */
+    private archivePartitionBucketed;
+    /**
+     * Common chunk-write flow: PUT data, verify, PUT manifest, transactional
+     * mark-archived-and-delete on the adapter, increment metrics.
+     *
+     * Shared by `archivePartitionLegacy` (v1 manifests) and
+     * `archivePartitionBucketed` (v2 manifests with bucket + maxColdDays).
+     */
+    private writeChunk;
+    private putChunkData;
+    private verifyChunkData;
+    private putManifest;
+    fetchRange<TRow>(args: ColdStoreFetchRangeArgs<TRow>): AsyncIterable<TRow>;
+    hasRange(args: Omit<ColdStoreFetchRangeArgs<unknown>, 'decode'>): Promise<boolean>;
+    countRange(args: Omit<ColdStoreFetchRangeArgs<unknown>, 'decode'>): Promise<number>;
+    /**
+     * Resolve all manifest sidecars whose time-range overlaps
+     * [fromTs, toTs). Uses one LIST walk under the tenant prefix, then
+     * GETs each `.manifest.json`.
+     */
+    private listRelevantManifests;
+    private getManifest;
+    private getChunkData;
+    /**
+     * Promote every row in a chunk back into PG. The adapter owns the
+     * actual INSERT + audit + chunk-counter decrement (transactional).
+     *
+     * Throws when:
+     *  - the cold store is disabled
+     *  - the adapter is unregistered or doesn't implement `replayInsert`
+     *  - the manifest is missing
+     *  - the data file's sha256 doesn't match the manifest's
+     *    `contentHash` (refuse to replay a tampered chunk)
+     *
+     * Idempotent on re-run: the adapter's `ON CONFLICT DO NOTHING`
+     * collapses the second pass to `inserted=0, skipped=rowCount`.
+     */
+    replayChunk(args: ColdStoreReplayChunkArgs): Promise<ColdStoreReplayResult>;
+    /**
+     * Locate the chunk whose `[minRowId, maxRowId]` window covers
+     * `rowId` and replay it. Scans manifests under the tenant prefix
+     * (LIST is one round-trip; manifests are tiny). When several manifests
+     * match (overlapping ranges from re-archives), the one with the
+     * smallest range is preferred — it's the most recently re-archived
+     * shard and therefore the one whose row contents are freshest.
+     *
+     * Returns `chunkId: null` when no manifest matches — the caller
+     * decides whether that means "row never existed" (rerun → 404) or
+     * "S3 outage" (rerun → 410 replayFailed). This method only fails
+     * loudly on infrastructure errors (missing chunk after manifest
+     * found, hash mismatch).
+     */
+    replayRow(args: ColdStoreReplayRowArgs): Promise<ColdStoreReplayResult>;
+    private toDate;
+    /**
+     * Subclass hook — query the per-DB `cold_store_chunks` index for
+     * purge candidates. Concrete subclasses (PlatformColdStore /
+     * OrchestratorColdStore) implement this with one Kysely SQL query
+     * keyed off `archived_at + max_cold_days * INTERVAL '1 day' < now()`
+     * and `max_cold_days != 'forever'`.
+     *
+     * Returning an empty array makes `purgeExpiredChunks` a no-op.
+     * Default implementation returns `[]` so non-purge subclasses (or
+     * test fixtures) can opt out trivially.
+     */
+    protected listPurgeableChunks(_opts: {
+        tableFilter?: string;
+        bucketFilter?: string;
+        limit: number;
+    }): Promise<PurgeableChunk[]>;
+    /**
+     * Acquire a per-chunk advisory lock so concurrent sweeps can't
+     * race on the same `DeleteObject` + `cold_store_chunks` row.
+     *
+     * Default implementation is a no-op (always succeeds). Subclasses
+     * with a Kysely instance override to use `pg_try_advisory_lock` on
+     * `hashtext('cold-store-purge|<db>|<table>|<chunkId>')` and release
+     * inside `fn`'s `finally`. Returns `null` when another worker holds
+     * the lock; returns the `fn` result on success.
+     */
+    protected withPurgeLock<T>(_args: {
+        table: string;
+        chunkId: string;
+    }, fn: () => Promise<T>): Promise<T | null>;
+    purgeExpiredChunks(opts?: PurgeExpiredChunksOpts): Promise<PurgeExpiredChunksSummary>;
+}
+//# sourceMappingURL=cold-store.d.ts.map