@voltro/plugin-cdc-out 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,296 @@
+import { DataStore } from '@voltro/database';
+import { Effect } from 'effect';
+import { PluginChangeEvent } from '@voltro/protocol';
+import { Row } from '@voltro/database';
+import { Schema } from 'effect';
+import { TableLike } from '@voltro/database';
+import { VoltroPlugin } from '@voltro/protocol';
+
+/**
+ * Thrown at plugin-construction time (= `app.config.ts` load = boot) when the
+ * cdc-out config is malformed: no sinks, a sink without `deliver`, a duplicate
+ * `table` in one instance, a non-positive numeric option, or an invalid
+ * webhook url. Boot aborts loudly — misconfig never ships silently.
+ */
+export declare class CdcConfigError extends CdcConfigError_base {
+}
+
+declare const CdcConfigError_base: Schema.TaggedErrorClass<CdcConfigError, "CdcConfigError", {
+    readonly _tag: Schema.tag<"CdcConfigError">;
+} & {
+    /** Human-readable detail, naming the offending `sinks[i]` / option / value. */
+    message: typeof Schema.String;
+}>;
+
+/**
+ * A single delivery attempt to a sink failed — a thrown sink error, a non-2xx
+ * webhook response, or the per-attempt `deliveryTimeoutMs` elapsing. The
+ * engine catches it, records `lastError` on the affected outbox rows, and
+ * schedules the retry (or dead-letters after `maxAttempts`).
+ */
+export declare class CdcDeliveryError extends CdcDeliveryError_base {
+    get message(): string;
+}
+
+declare const CdcDeliveryError_base: Schema.TaggedErrorClass<CdcDeliveryError, "CdcDeliveryError", {
+    readonly _tag: Schema.tag<"CdcDeliveryError">;
+} & {
+    /** The sink's `name`. */
+    sink: typeof Schema.String;
+    /** Human-readable failure detail (status line, timeout, thrown message). */
+    reason: typeof Schema.String;
+}>;
+
+/**
+ * One instance-scoped lease, heartbeat-renewed. `isLeader` is the in-memory
+ * view the tap + workers consult; it flips false the moment a heartbeat fails
+ * (fail closed: no lease, no enqueue, no delivery).
+ *
+ * Correctness mirrors the scheduler's claims table: every transition is ONE
+ * atomic statement, so two replicas racing resolve to exactly one holder —
+ *   renew:    UPDATE … WHERE key = ? AND holder = me
+ *   bootstrap: INSERT … ON CONFLICT (key) DO NOTHING
+ *   takeover: UPDATE … WHERE key = ? AND holder = <seen> AND expiresAt <= now
+ * The standard lease caveat applies (a paused-then-resumed old leader can
+ * overlap for up to one heartbeat) — which is why delivery claims are ALSO
+ * CAS-guarded per row in the engine.
+ */
+export declare class CdcLeaseManager {
+    private readonly opts;
+    private held;
+    constructor(opts: {
+        readonly key: string;
+        readonly holder: string;
+        readonly ttlMs: number;
+        readonly now?: () => number;
+    });
+    get isLeader(): boolean;
+    /** Acquire or renew the lease. Returns (and records) whether THIS process
+     *  holds it. Any store error → fail closed (false). */
+    heartbeat(store: DataStore): Promise<boolean>;
+    /** Give the lease up (graceful shutdown) — expire it so failover is
+     *  immediate instead of waiting out the TTL. */
+    release(store: DataStore): Promise<void>;
+}
+
+export declare const CDCOUT_LEASES_TABLE = "_voltro_cdcout_leases";
+
+export declare const CDCOUT_OUTBOX_TABLE = "_voltro_cdcout_outbox";
+
+export declare class CdcOutEngine {
+    private readonly opts;
+    private readonly now;
+    /** Rows stuck in `delivering` longer than this are treated as a crashed
+     *  claimant's and returned to `pending`. A LIVE worker always resolves a
+     *  batch within `deliveryTimeoutMs`, so 2× (floored) can never steal from
+     *  a live one. */
+    private readonly stuckClaimMs;
+    constructor(opts: CdcOutEngineOptions);
+    /** One sweep over every pipe. Leader-gated (fail closed): a non-leader
+     *  replica's workers never touch the outbox. */
+    tick(store: DataStore): Promise<void>;
+    private pipeTick;
+}
+
+export declare interface CdcOutEngineOptions {
+    readonly pipes: ReadonlyArray<EnginePipe>;
+    readonly replicaId: string;
+    readonly maxAttempts: number;
+    readonly backoffBaseMs: number;
+    readonly backoffMaxMs: number;
+    readonly deliveryTimeoutMs: number;
+    /** Injectable clock (tests). Default `Date.now`. */
+    readonly now?: () => number;
+    /** Injectable jitter source (tests). Default `Math.random`. */
+    readonly random?: () => number;
+    readonly isLeader: () => boolean;
+    readonly warn?: (message: string, fields?: Record<string, unknown>) => void;
+}
+
+export declare const cdcOutLeasesTable: TableLike;
+
+export declare interface CdcOutOptions {
+    readonly sinks: ReadonlyArray<SinkConfig>;
+    /** Delivery attempts per record before it dead-letters (default 5). */
+    readonly maxAttempts?: number;
+    /**
+     * Instance name — REQUIRED when wiring the plugin more than once (e.g. two
+     * sinks for one table = two instances). Suffixes the plugin name
+     * (`@voltro/plugin-cdc-out#analytics`), the inspect mount
+     * (`…/plugins/cdc-out--analytics/…`) and the outbox pipe keys.
+     */
+    readonly name?: string;
+    /** First retry delay; doubles per attempt with jitter (default 200). */
+    readonly backoffBaseMs?: number;
+    /** Retry delay ceiling (default 30_000). */
+    readonly backoffMaxMs?: number;
+    /** Outbox poll cadence of the delivery workers (default 250). */
+    readonly sweepIntervalMs?: number;
+    /** Per-attempt delivery timeout — aborts the sink call (default 10_000). */
+    readonly deliveryTimeoutMs?: number;
+    /** Leader-lease TTL; heartbeat renews at ttl/3 (default 15_000). */
+    readonly leaseTtlMs?: number;
+    /** Retention for delivered/dead outbox rows, in hours (default: the
+     *  `CDCOUT_RETENTION_HOURS` env var, else 72). */
+    readonly retentionHours?: number;
+}
+
+export declare const cdcOutOutboxTable: TableLike;
+
+export declare const cdcOutPlugin: (options: CdcOutOptions) => VoltroPlugin;
+
+/**
+ * A delivery target. `deliver` may return a Promise OR an Effect — async
+ * connector authors and Effect-native ones both compose without wrapping.
+ * A throw / rejection / Effect failure marks the attempt failed; the engine
+ * retries with backoff and dead-letters after `maxAttempts`.
+ */
+export declare interface CdcSink {
+    readonly name: string;
+    readonly deliver: (batch: ReadonlyArray<SinkRecord>, ctx: DeliverContext) => Promise<void> | Effect.Effect<void, unknown>;
+    /** Host the sink talks to — surfaced as a `network:outbound:<host>` permission. */
+    readonly outboundHost?: string;
+}
+
+/** Passed to `deliver` — abort fires when the attempt times out
+ *  (`deliveryTimeoutMs`) so an HTTP sink can cancel its request. */
+export declare interface DeliverContext {
+    readonly signal: AbortSignal;
+}
+
+/**
+ * Run ONE delivery attempt against a sink: supports Promise- and
+ * Effect-returning `deliver` implementations, fails typed
+ * (`CdcDeliveryError`), times out after `timeoutMs` (aborting the handed-in
+ * `AbortSignal` so a hung HTTP call is truly cancelled), and wraps the
+ * attempt in a span.
+ */
+export declare const deliverOnce: (sink: CdcSink, batch: ReadonlyArray<SinkRecord>, opts: {
+    readonly timeoutMs: number;
+    readonly pipe: string;
+}) => Effect.Effect<void, CdcDeliveryError>;
+
+export declare interface EnginePipe {
+    /** Pipe identity — `<instance>:<table>:<sink name>` (the outbox `pipe` column). */
+    readonly key: string;
+    readonly sink: CdcSink;
+    readonly batchSize: number;
+    /** In-process serialisation: one in-flight batch per pipe. */
+    busy: boolean;
+    readonly stats: PipeStats;
+}
+
+/**
+ * Seed a pipe with existing rows (mirror of search's `backfillIndex`): every
+ * current row is enqueued through the SAME durable outbox as live changes —
+ * delivered in order, retried, dead-lettered, with a stable per-row
+ * `deliveryKey`. Call from a `*.startup.tsx` / CLI with the app's store.
+ * Rows without an `id` are skipped. Returns the number enqueued.
+ */
+export declare const enqueueBackfill: (store: DataStore, config: SinkConfig, rows: ReadonlyArray<Record<string, unknown>>, opts?: {
+    readonly instanceName?: string;
+    readonly now?: () => number;
+}) => Promise<number>;
+
+/** Map one ChangeEvent through a config's filter + map, or null when
+ *  filtered / unkeyable. */
+export declare const mapChange: (event: PluginChangeEvent, config: SinkConfig) => MappedChange | null;
+
+/** A change mapped through a pipe's filter + map, ready to enqueue. */
+export declare interface MappedChange {
+    readonly table: string;
+    readonly op: 'insert' | 'update' | 'delete';
+    readonly key: string;
+    readonly data: Record<string, unknown> | null;
+}
+
+/** In-process sink — records every delivered batch. For dev + tests. */
+export declare const memorySink: (name?: string) => CdcSink & {
+    readonly batches: ReadonlyArray<ReadonlyArray<SinkRecord>>;
+    readonly delivered: ReadonlyArray<SinkRecord>;
+};
+
+/** Generate the next outbox id (== deliveryKey). Called SYNCHRONOUSLY in the
+ *  change tap, so ids carry commit order (uuidv7 is monotonic per process). */
+export declare const nextOutboxId: () => string;
+
+export declare interface OutboxRow {
+    readonly id: string;
+    readonly pipe: string;
+    readonly sourceTable: string;
+    readonly op: 'insert' | 'update' | 'delete';
+    readonly key: string;
+    readonly payload: Record<string, unknown> | null;
+    readonly status: OutboxStatus;
+    readonly attempts: number;
+    readonly nextAttemptAt: Date;
+    readonly claimedBy: string | null;
+    readonly claimedAt: Date | null;
+    readonly createdAt: Date;
+    readonly deliveredAt: Date | null;
+    readonly lastError: string | null;
+}
+
+/** Build one pending outbox row for a mapped change. The freshly generated
+ *  TypeID `id` is the record's `deliveryKey`. */
+export declare const outboxRowFor: (mapped: MappedChange, pipe: string, nowMs: number) => Row;
+
+export declare type OutboxStatus = 'pending' | 'delivering' | 'delivered' | 'dead';
+
+/** Pipe identity — `<instance>:<table>:<sink name>` (the outbox `pipe` column). */
+export declare const pipeKey: (instanceName: string, config: SinkConfig) => string;
+
+export declare interface PipeStats {
+    delivered: number;
+    failed: number;
+    dead: number;
+    lastDeliveryAtMs: number | null;
+}
+
+/** Exponential backoff with full-range-of-upper-half jitter:
+ *  `min(base·2^(attempts-1), max) × (0.5 + 0.5·random)`. */
+export declare const retryDelayMs: (attempts: number, opts: {
+    readonly baseMs: number;
+    readonly maxMs: number;
+    readonly random?: () => number;
+}) => number;
+
+export declare interface SinkConfig {
+    readonly table: string;
+    readonly sink: CdcSink;
+    /** Map a row → the outbound record body. Default: the row unchanged. */
+    readonly map?: (row: Record<string, unknown>) => Record<string, unknown>;
+    /** Skip a change when this returns false. */
+    readonly filter?: (event: PluginChangeEvent) => boolean;
+    /** Max records per delivered batch (default 100). */
+    readonly batchSize?: number;
+}
+
+export declare interface SinkRecord {
+    readonly table: string;
+    readonly op: 'insert' | 'update' | 'delete';
+    /** The source row's id. */
+    readonly key: string;
+    /** The mapped record body (null on delete). */
+    readonly data: Record<string, unknown> | null;
+    /**
+     * At-least-once dedupe handle: the record's durable outbox id (a TypeID —
+     * unique across replicas, stable across restarts and retries). A retried
+     * batch re-sends the SAME keys; a sink that upserts on `deliveryKey`
+     * collapses duplicates safely.
+     */
+    readonly deliveryKey: string;
+}
+
+/** Validate config — loud, typed boot error (`CdcConfigError`) on anything
+ *  malformed, including a DUPLICATE `table` (never silent last-wins). */
+export declare const validateCdcOutConfig: (options: CdcOutOptions) => void;
+
+/** POST each batch as JSON to a webhook. The simplest real sink. Honors the
+ *  engine's per-attempt abort signal, so a hung connection is cancelled when
+ *  the delivery times out. */
+export declare const webhookSink: (url: string, init?: {
+    readonly headers?: Record<string, string>;
+}) => CdcSink;
+
+export { }