@voltro/plugin-cdc-out 0.32.0 → 0.34.0

package/dist/index.d.ts

@@ -6,6 +6,14 @@ import { Schema } from 'effect';
 import { TableLike } from '@voltro/database';
 import { VoltroPlugin } from '@voltro/protocol';
 
+/**
+ * Deterministic JSON — object keys sorted at every depth, so two replicas
+ * that decoded the same row into differently-ordered objects still digest
+ * identically. `Date`s normalise to ISO (a binlog reader hands back `Date`s
+ * where a NOTIFY payload hands back strings), `undefined` to null.
+ */
+export declare const canonicalJson: (value: unknown) => string;
+
 /**
  * Thrown at plugin-construction time (= `app.config.ts` load = boot) when the
  * cdc-out config is malformed: no sinks, a sink without `deliver`, a duplicate
@@ -73,10 +81,25 @@ export declare class CdcLeaseManager {
     release(store: DataStore): Promise<void>;
 }
 
+export declare const CDCOUT_CLAIMS_TABLE = "_voltro_cdcout_claims";
+
 export declare const CDCOUT_LEASES_TABLE = "_voltro_cdcout_leases";
 
 export declare const CDCOUT_OUTBOX_TABLE = "_voltro_cdcout_outbox";
 
+/**
+ * The enqueue idempotence key. `unique(pipe, changeKey)` is the whole point:
+ * an enqueue is an `insertIgnore` here FIRST, and only the replica whose
+ * `claimedBy` comes back writes the outbox row. That is what lets every
+ * replica try during a leadership gap without producing duplicates.
+ *
+ * `claimedBy` also makes the claim RE-ENTRANT for its owner: a replica that
+ * claimed and then failed the outbox insert reads its own id back on the next
+ * drain and finishes the job, instead of treating its own claim as somebody
+ * else's and dropping the change.
+ */
+export declare const cdcOutClaimsTable: TableLike;
+
 export declare class CdcOutEngine {
     private readonly opts;
     private readonly now;
@@ -119,6 +142,24 @@ export declare interface CdcOutOptions {
      * (`@voltro/plugin-cdc-out#analytics`), the inspect mount
      * (`…/plugins/cdc-out--analytics/…`) and the outbox pipe keys.
      */
+    /**
+     * Namespace for this plugin's inspect endpoints. Default `cdc-out`.
+     *
+     * Set it when your app already publishes under that name — an exact tag
+     * collision is fatal at codegen, and this is the way out. Orthogonal to
+     * `name` below: `alias` REPLACES the namespace, `name` distinguishes two
+     * installations within it.
+     *
+     * The cost, stated because nothing else states it: the local and cloud
+     * dashboards fetch this plugin's panel at the DEFAULT slug, so an aliased
+     * install keeps working while its dashboard panel 404s. Alias to escape a
+     * collision, not for taste.
+     */
+    readonly alias?: string;
+    /**
+     * Discriminator for a SECOND installation of this plugin, when one app runs
+     * two (`@voltro/plugin-cdc-out#analytics`). Not a rename — for that use `alias`.
+     */
     readonly name?: string;
     /** First retry delay; doubles per attempt with jitter (default 200). */
     readonly backoffBaseMs?: number;
@@ -133,6 +174,27 @@ export declare interface CdcOutOptions {
     /** Retention for delivered/dead outbox rows, in hours (default: the
      *  `CDCOUT_RETENTION_HOURS` env var, else 72). */
     readonly retentionHours?: number;
+    /**
+     * How long an enqueue claim (`_voltro_cdcout_claims`) is kept — the window
+     * inside which two replicas racing to enqueue the same fleet change
+     * collapse to one row. It only has to outlive a leadership handoff, so the
+     * default is `max(60_000, 4 × leaseTtlMs)`.
+     *
+     * Raising it costs one small row per change for longer and widens the span
+     * over which two byte-identical changes to the same row are told apart by
+     * their occurrence counter rather than by content. Lowering it below
+     * `leaseTtlMs + one heartbeat` is what re-opens the duplicate window.
+     */
+    readonly dedupWindowMs?: number;
+    /** How far back each replica keeps observed fleet changes in memory for a
+     *  takeover to drain (default: `dedupWindowMs`). A handoff that takes longer
+     *  than this loses the changes older than the window — reported as
+     *  `droppedPending` on the `/sinks` inspect endpoint. */
+    readonly handoffBufferMs?: number;
+    /** Hard ceiling on buffered fleet changes per instance — oldest dropped
+     *  first (default 10_000). Bounds the memory a write burst during a
+     *  leadership gap can cost. */
+    readonly handoffBufferSize?: number;
 }
 
 export declare const cdcOutOutboxTable: TableLike;
@@ -152,6 +214,20 @@ export declare interface CdcSink {
     readonly outboundHost?: string;
 }
 
+/** The content half of a change identity — equal iff two replicas are looking
+ *  at the same change (or at two changes with byte-identical images, which the
+ *  occurrence counter then separates). */
+export declare const changeDigest: (input: {
+    readonly pipe: string;
+    readonly op: "insert" | "update" | "delete";
+    readonly key: string;
+    readonly next: Record<string, unknown> | null;
+    readonly prev: Record<string, unknown> | null;
+}) => string;
+
+/** Compose the durable identity written to `_voltro_cdcout_claims.changeKey`. */
+export declare const composeChangeKey: (digest: string, occurrence: number) => string;
+
 /** Passed to `deliver` — abort fires when the attempt times out
  *  (`deliveryTimeoutMs`) so an HTTP sink can cancel its request. */
 export declare interface DeliverContext {
@@ -192,6 +268,41 @@ export declare const enqueueBackfill: (store: DataStore, config: SinkConfig, row
     readonly now?: () => number;
 }) => Promise<number>;
 
+export declare class HandoffBuffer<T> {
+    #private;
+    constructor(opts: {
+        readonly windowMs: number;
+        readonly maxEntries: number;
+    });
+    get size(): number;
+    /** Entries evicted (by age or ceiling) while still un-enqueued — the honest
+     *  count of changes the buffer could NOT carry across a handoff. */
+    get droppedPending(): number;
+    append(entry: HandoffEntry<T>): void;
+    /** The oldest entry still awaiting enqueue, or undefined. Drained in
+     *  observation order so the outbox ids stay commit-ordered. */
+    nextPending(): HandoffEntry<T> | undefined;
+    pendingCount(): number;
+    /** Forget entries older than the window (already-enqueued ones silently,
+     *  un-enqueued ones counted). */
+    prune(nowMs: number): void;
+}
+
+/** One observed change, waiting to be enqueued (or already enqueued). */
+export declare interface HandoffEntry<T> {
+    /** Pipe key — `<instance>:<table>:<sink name>`. */
+    readonly pipe: string;
+    /** The durable change identity this entry will be claimed under. Computed
+     *  at OBSERVATION time, so re-draining an entry re-uses the same key and
+     *  collapses instead of duplicating. */
+    readonly changeKey: string;
+    readonly change: T;
+    readonly observedAtMs: number;
+    /** Set once the entry has been resolved — enqueued by us, or found already
+     *  claimed by another replica. */
+    enqueued: boolean;
+}
+
 /** Map one ChangeEvent through a config's filter + map, or null when
  *  filtered / unkeyable. */
 export declare const mapChange: (event: PluginChangeEvent, config: SinkConfig) => MappedChange | null;
@@ -210,10 +321,46 @@ export declare const memorySink: (name?: string) => CdcSink & {
     readonly delivered: ReadonlyArray<SinkRecord>;
 };
 
+/** Generate the next enqueue-claim id. */
+export declare const nextClaimId: () => string;
+
 /** Generate the next outbox id (== deliveryKey). Called SYNCHRONOUSLY in the
  *  change tap, so ids carry commit order (uuidv7 is monotonic per process). */
 export declare const nextOutboxId: () => string;
 
+/**
+ * How many times this replica has already seen each digest inside the dedup
+ * window. Bounded on BOTH axes — a per-change map with no ceiling is an
+ * unbounded table in memory:
+ *   * by time — entries untouched for `windowMs` are pruned, and
+ *   * by size — `maxEntries` oldest-first, so a burst of unique changes can't
+ *     grow it without limit.
+ *
+ * `windowMs` MUST be longer than the lifetime of a claim row in the database.
+ * That ordering is the invariant: if a claim outlived the counter that
+ * produced it, the next sighting of that digest would restart at occurrence 0,
+ * collide with the retained claim, and be dropped as a duplicate it is not.
+ */
+export declare class OccurrenceCounter {
+    #private;
+    constructor(opts: {
+        readonly windowMs: number;
+        readonly maxEntries: number;
+    });
+    get size(): number;
+    /** Record a sighting and return its occurrence index (0 for the first). */
+    next(digest: string, nowMs: number): number;
+    /**
+     * Align this replica with the fleet from the claims already recorded in the
+     * database — the counters of a process that booted mid-window would
+     * otherwise restart at 0 and re-key changes the incumbents have already
+     * keyed. Call ONCE, before the first observed change is keyed.
+     */
+    seedFromChangeKeys(changeKeys: Iterable<string>, nowMs: number): void;
+    /** Drop entries whose last sighting is older than the window. */
+    prune(nowMs: number): void;
+}
+
 export declare interface OutboxRow {
     readonly id: string;
     readonly pipe: string;
@@ -237,6 +384,12 @@ export declare const outboxRowFor: (mapped: MappedChange, pipe: string, nowMs: n
 
 export declare type OutboxStatus = 'pending' | 'delivering' | 'delivered' | 'dead';
 
+/** Split a stored `changeKey` back into its two halves (`null` when malformed). */
+export declare const parseChangeKey: (changeKey: string) => {
+    digest: string;
+    occurrence: number;
+} | null;
+
 /** Pipe identity — `<instance>:<table>:<sink name>` (the outbox `pipe` column). */
 export declare const pipeKey: (instanceName: string, config: SinkConfig) => string;