@nodii/telemetry 0.13.0 → 0.14.1

package/dist/worker/outbox-dispatcher.js

@@ -0,0 +1,285 @@
+// D402 — OutboxDispatcher: the Pattern-3 PRODUCER drain.
+//
+// D402 (LOCKED): "OutboxDispatcher — Pattern-3 producer drain." It GENERALIZES
+// the pattern shipped in `ts/outbox-dispatcher/src/drainer.ts` (which this lib
+// does NOT edit — outbox-dispatcher gets re-based onto this substrate in
+// Phase 2). The producer side of the both-sides-completeness rule: every outbox
+// topic has a standardized producer (this) AND a standardized consumer
+// (StreamsWorker / ReplicationWorker).
+//
+// ── Composition (which primitives, how) ──────────────────────────────────────
+//   - SingleActiveLease  — single-active per (service, outbox) so two dispatcher
+//     replicas don't both drain. ttl/2 renew inside the lease. `validateFence`
+//     guards the mark-dispatched write (a lease-lost dispatcher's late mark is
+//     rejected — it must not mark a row another dispatcher is publishing).
+//   - CheckpointStore    — tracks the HWM (the last published stream id) so the
+//     `nodii.worker.current_hwm` gauge + resume position are durable. The mark
+//     UPDATE and the cursor advance run in the SAME claim tx (atomic).
+//   - ObservabilityHooks — lease/hwm/depth/drain metrics, canonical names.
+//   - SignalDrain        — `stop({timeoutMs})` registerable; bounded drain.
+//   - DeadLetterSink     — OPTIONAL and currently NOT invoked on any path. Per
+//     D401 the producer drain has NO source-side dead-letter: a terminal /
+//     unpublishable row is surfaced via `recordFailure` (keep-row,
+//     publish_attempts bumped) + the 24h-unpublished sweep, NOT a sink. The sink
+//     is accepted for API symmetry with the consumer workers + a future opt-in
+//     (PII-redacted if provided), but no drain path calls it today.
+//
+// ── The drain sequence (per 01-comms § 9.2 / outbox-dispatcher § 5.2) ─────────
+//   acquire-lease → claim a batch (dispatched_at IS NULL, occurred_at order,
+//   FOR UPDATE SKIP LOCKED) inside a tx → publish each (injected publisher) →
+//   mark dispatched (fence-guarded) + advance the HWM cursor, all in the SAME
+//   tx → commit. D401: a publish failure increments attempts + leaves the row in
+//   place (never DLQ).
+//
+// ── At-least-once semantics (NOT exactly-once on the producer side) ───────────
+//   The PUBLISH (XADD) runs BEFORE `validateFence` — it is NOT fenced. A stale or
+//   retried dispatcher's publish therefore DOES land: the producer is
+//   AT-LEAST-ONCE (the same row may be re-published after a crash-before-commit
+//   or a lease handoff). The fence guards only the mark-dispatched UPDATE — it
+//   stops a lease-lost dispatcher from falsely recording a row as dispatched
+//   (which would suppress the legitimate successor's re-publish). END-TO-END
+//   exactly-once is achieved by the CONSUMER's `consumed_events` dedupe (D198),
+//   NOT by the fence preventing the publish.
+import { ensurePiiRedaction } from "./dead-letter.js";
+import { hwmGaugeValue, ObservabilityHooks } from "./observability-hooks.js";
+import { FenceCheckUnavailableError, FenceLostError, SingleActiveLease, } from "./single-active-lease.js";
+const DEFAULT_BATCH_SIZE = 100;
+const DEFAULT_POLL_MS = 1_000;
+const DEFAULT_STOP_TIMEOUT_MS = 30_000;
+/**
+ * OutboxDispatcher — the Pattern-3 producer drain. Composes lease + checkpoint +
+ * hooks + drain. Single-active; the mark-dispatched + cursor advance are atomic
+ * in the claim tx; publish is at-least-once (the consumer dedupes).
+ */
+export class OutboxDispatcher {
+    publish;
+    begin;
+    claimBatch;
+    markDispatched;
+    recordFailure;
+    checkpoint;
+    lease;
+    subscription;
+    // OPTIONAL + currently uninvoked (see the opts docstring). Kept for API
+    // symmetry / a future opt-in; PII-redacted if provided.
+    deadLetter;
+    batchSize;
+    pollMs;
+    hooks;
+    autoStart;
+    _running = false;
+    _stopped = false;
+    _inFlight = 0;
+    _hwm = { lastStreamId: null };
+    _lastAppliedAtMs = null;
+    loop = Promise.resolve();
+    constructor(opts) {
+        this.publish = opts.publish;
+        this.begin = opts.begin;
+        this.claimBatch = opts.claimBatch;
+        this.markDispatched = opts.markDispatched;
+        this.recordFailure = opts.recordFailure;
+        this.checkpoint = opts.checkpoint;
+        this.lease = new SingleActiveLease({
+            ...opts.lease,
+            onLeaseLost: () => this.hooks.leaseLost(),
+        });
+        this.subscription = opts.subscription ?? `outbox:${opts.lease.service}`;
+        this.deadLetter = opts.deadLetter
+            ? ensurePiiRedaction(opts.deadLetter)
+            : undefined;
+        this.batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
+        this.pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
+        this.autoStart = opts.autoStart !== false;
+        this.hooks =
+            opts.hooks ??
+                new ObservabilityHooks({
+                    attributes: {
+                        service: opts.lease.service,
+                        scope: opts.lease.scope,
+                        worker: opts.workerLabel ?? "outbox",
+                    },
+                });
+    }
+    get running() {
+        return this._running;
+    }
+    get leaseAcquired() {
+        return this.lease.leaseAcquired;
+    }
+    get lag() {
+        if (this._lastAppliedAtMs === null)
+            return null;
+        return Math.max(0, (Date.now() - this._lastAppliedAtMs) / 1000);
+    }
+    get currentHwm() {
+        return this._hwm;
+    }
+    async start() {
+        if (this._running)
+            return;
+        this._stopped = false;
+        const acquired = await this.lease.acquire();
+        if (acquired)
+            this.hooks.leaseAcquired();
+        await this.hydrateCursor();
+        this._running = true;
+        // Background drain loop. `autoStart:false` lets a test drive `drainOnce()`.
+        if (this.autoStart)
+            this.loop = this.runLoop();
+    }
+    async hydrateCursor() {
+        try {
+            await this.begin(async (tx) => {
+                const cp = await this.checkpoint.get(this.subscription, tx);
+                if (cp)
+                    this._hwm = { lastStreamId: cp.lastStreamId };
+            });
+        }
+        catch {
+            /* cold start — cursor stays null */
+        }
+    }
+    async runLoop() {
+        while (!this._stopped) {
+            try {
+                if (!this.lease.leaseAcquired) {
+                    const got = await this.lease.acquire();
+                    if (got)
+                        this.hooks.leaseAcquired();
+                    else {
+                        await new Promise((r) => setTimeout(r, this.pollMs));
+                        continue;
+                    }
+                }
+                const n = await this.drainOnce();
+                if (n < this.batchSize) {
+                    await new Promise((r) => setTimeout(r, this.pollMs));
+                }
+            }
+            catch (err) {
+                // A drain-level error (claim tx itself rejected — Postgres down) is the
+                // Pillar-5 infra-failure case: back off, NEVER crash the loop.
+                this.hooks.recordDepth(-1, { error: "drain_failed" });
+                void err;
+                await new Promise((r) => setTimeout(r, Math.min(this.pollMs, 200)));
+            }
+        }
+    }
+    /**
+     * One drain pass: claim a batch in ONE tx, publish each, mark dispatched +
+     * advance the HWM cursor (fence-guarded) in the SAME tx. Returns the count
+     * dispatched. The publish runs INSIDE the claim tx so the row-lock is held
+     * across it (a second dispatcher can't double-claim); at-least-once safe (a
+     * crash after publish but before commit re-publishes; the consumer dedupes).
+     */
+    async drainOnce() {
+        if (!this.lease.leaseAcquired) {
+            const got = await this.lease.acquire();
+            if (got)
+                this.hooks.leaseAcquired();
+            else
+                return 0;
+        }
+        const token = this.lease.fencingToken;
+        let dispatched = 0;
+        let lastStreamId = null;
+        await this.begin(async (tx) => {
+            const rows = await this.claimBatch(tx, this.batchSize);
+            if (rows.length === 0)
+                return;
+            for (const row of rows) {
+                if (this.lease.lost)
+                    break;
+                this._inFlight += 1;
+                let fenceLost = false;
+                try {
+                    const streamId = await this.publish(row);
+                    // The fence guard BEFORE marking the row dispatched: a lease-lost
+                    // dispatcher must not mark a row a successor is also publishing.
+                    if (token === null) {
+                        throw new FenceLostError({
+                            leaseKey: this.lease.key,
+                            expectedToken: -1,
+                            current: null,
+                        });
+                    }
+                    await this.lease.validateFence(token);
+                    await this.markDispatched(tx, row.id, streamId);
+                    if (streamId)
+                        lastStreamId = streamId;
+                    dispatched += 1;
+                    this._lastAppliedAtMs = Date.now();
+                }
+                catch (err) {
+                    if (err instanceof FenceLostError ||
+                        err instanceof FenceCheckUnavailableError) {
+                        // The lease was lost/taken-over (FenceLostError) OR the fence-check
+                        // READ failed transiently (FenceCheckUnavailableError — lease-redis
+                        // blip). Either way ABORT the rest of the batch WITHOUT recordFailure:
+                        // a transient lease-redis blip is NOT a publish failure, so it must
+                        // NOT bump publish_attempts (5 blips would otherwise silently abandon
+                        // a row whose every publish succeeded). The row is retried on lease
+                        // recovery; the tx commits only the rows marked BEFORE the loss (each
+                        // fence-validated). Mirrors Streams/Replication + the Go dispatcher.
+                        this.hooks.leaseLost();
+                        fenceLost = true;
+                    }
+                    else {
+                        // D401: a publish failure NEVER deletes / DLQs the row — record the
+                        // failure (bump attempts) + leave it. The 24h-unpublished sweep
+                        // surfaces persistent liveness problems.
+                        const msg = err instanceof Error ? err.message : String(err);
+                        if (this.recordFailure) {
+                            await this.recordFailure(tx, row.id, msg);
+                        }
+                    }
+                }
+                finally {
+                    this._inFlight -= 1;
+                }
+                if (fenceLost)
+                    break;
+            }
+            // Advance the HWM cursor in the SAME tx as the marks (atomic): the cursor
+            // can never advance past a row that wasn't actually marked dispatched.
+            if (lastStreamId !== null) {
+                await this.checkpoint.advance({ subscription: this.subscription, lastStreamId }, tx);
+            }
+        });
+        if (lastStreamId !== null) {
+            this._hwm = { lastStreamId };
+            this.hooks.recordHwm(hwmGaugeValue(lastStreamId));
+        }
+        this.hooks.recordDepth(dispatched);
+        return dispatched;
+    }
+    /** Run exactly one drain pass (test seam / cron tick). */
+    async runOnce() {
+        return this.drainOnce();
+    }
+    async stop(opts) {
+        const timeoutMs = opts?.timeoutMs ?? DEFAULT_STOP_TIMEOUT_MS;
+        const start = Date.now();
+        this._stopped = true;
+        this._running = false;
+        try {
+            await Promise.race([
+                this.loop,
+                new Promise((r) => setTimeout(r, timeoutMs)),
+            ]);
+        }
+        catch {
+            /* swallow */
+        }
+        while (this._inFlight > 0 && Date.now() - start < timeoutMs) {
+            await new Promise((r) => setTimeout(r, 10));
+        }
+        const drainedCleanly = this._inFlight === 0;
+        await this.lease.release();
+        const durationMs = Date.now() - start;
+        this.hooks.recordDrainDuration(durationMs);
+        return { drainedCleanly, durationMs };
+    }
+}
+//# sourceMappingURL=outbox-dispatcher.js.map

package/dist/worker/outbox-dispatcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"outbox-dispatcher.js","sourceRoot":"","sources":["../../src/worker/outbox-dispatcher.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,EAAE;AACF,+EAA+E;AAC/E,+EAA+E;AAC/E,yEAAyE;AACzE,gFAAgF;AAChF,uEAAuE;AACvE,uCAAuC;AACvC,EAAE;AACF,gFAAgF;AAChF,iFAAiF;AACjF,+EAA+E;AAC/E,+EAA+E;AAC/E,2EAA2E;AAC3E,gFAAgF;AAChF,+EAA+E;AAC/E,uEAAuE;AACvE,2EAA2E;AAC3E,4EAA4E;AAC5E,+EAA+E;AAC/E,2EAA2E;AAC3E,mEAAmE;AACnE,iFAAiF;AACjF,+EAA+E;AAC/E,oEAAoE;AACpE,EAAE;AACF,iFAAiF;AACjF,6EAA6E;AAC7E,8EAA8E;AAC9E,8EAA8E;AAC9E,iFAAiF;AACjF,uBAAuB;AACvB,EAAE;AACF,iFAAiF;AACjF,kFAAkF;AAClF,sEAAsE;AACtE,gFAAgF;AAChF,+EAA+E;AAC/E,6EAA6E;AAC7E,6EAA6E;AAC7E,gFAAgF;AAChF,6CAA6C;AAG7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC7E,OAAO,EACL,0BAA0B,EAC1B,cAAc,EACd,iBAAiB,GAElB,MAAM,0BAA0B,CAAC;AA0ElC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAEvC;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IACV,OAAO,CAAkB;IACzB,KAAK,CAEN;IACC,UAAU,CAGD;IACT,cAAc,CAIZ;IACF,aAAa,CAIX;IACF,UAAU,CAAkB;IAC5B,KAAK,CAAoB;IACzB,YAAY,CAAS;IACtC,wEAAwE;IACxE,wDAAwD;IACvC,UAAU,CAAkB;IAC5B,SAAS,CAAS;IAClB,MAAM,CAAS;IACf,KAAK,CAAqB;IAC1B,SAAS,CAAU;IAE5B,QAAQ,GAAG,KAAK,CAAC;IACjB,QAAQ,GAAG,KAAK,CAAC;IACjB,SAAS,GAAG,CAAC,CAAC;IACd,IAAI,GAAc,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACzC,gBAAgB,GAAkB,IAAI,CAAC;IACvC,IAAI,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEhD,YAAY,IAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;QACxC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,IAAI,iBAAiB,CAAC;YACjC,GAAG,IAAI,CAAC,KAAK;YACb,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;SAC1C,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,UAAU,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACxE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU;YAC/B,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC;YACrC,CAAC,CAAC,SAAS,CAAC;QACd,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;QACtD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,eAAe,CAAC;QAC7C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC;QAC1C,IAAI,CAAC,KAAK;YACR,IAAI,CAAC,KAAK;gBACV,IAAI,kBAAkB,CAAC;oBACrB,UAAU,EAAE;wBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;wBAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;wBACvB,MAAM,EAAE,IAAI,CAAC,WAAW,IAAI,QAAQ;qBACrC;iBACF,CAAC,CAAC;IACP,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;IAClC,CAAC;IAED,IAAI,GAAG;QACL,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC5C,IAAI,QAAQ;YAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;QACzC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,4EAA4E;QAC5E,IAAI,IAAI,CAAC,SAAS;YAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAC5B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAC5D,IAAI,EAAE;oBAAE,IAAI,CAAC,IAAI,GAAG,EAAE,YAAY,EAAE,EAAE,CAAC,YAAY,EAAE,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,oCAAoC;QACtC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;oBAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;oBACvC,IAAI,GAAG;wBAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;yBAC/B,CAAC;wBACJ,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;wBAC3D,SAAS;oBACX,CAAC;gBACH,CAAC;gBACD,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACvB,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,wEAAwE;gBACxE,+DAA+D;gBAC/D,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;gBACtD,KAAK,GAAG,CAAC;gBACT,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAC5B,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAC1C,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACvC,IAAI,GAAG;gBAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;;gBAC/B,OAAO,CAAC,CAAC;QAChB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC;QACtC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAkB,IAAI,CAAC;QACvC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YACvD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;oBAAE,MAAM;gBAC3B,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;gBACpB,IAAI,SAAS,GAAG,KAAK,CAAC;gBACtB,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBACzC,kEAAkE;oBAClE,iEAAiE;oBACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;wBACnB,MAAM,IAAI,cAAc,CAAC;4BACvB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG;4BACxB,aAAa,EAAE,CAAC,CAAC;4BACjB,OAAO,EAAE,IAAI;yBACd,CAAC,CAAC;oBACL,CAAC;oBACD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBACtC,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;oBAChD,IAAI,QAAQ;wBAAE,YAAY,GAAG,QAAQ,CAAC;oBACtC,UAAU,IAAI,CAAC,CAAC;oBAChB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACrC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IACE,GAAG,YAAY,cAAc;wBAC7B,GAAG,YAAY,0BAA0B,EACzC,CAAC;wBACD,oEAAoE;wBACpE,oEAAoE;wBACpE,uEAAuE;wBACvE,oEAAoE;wBACpE,sEAAsE;wBACtE,oEAAoE;wBACpE,sEAAsE;wBACtE,qEAAqE;wBACrE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;wBACvB,SAAS,GAAG,IAAI,CAAC;oBACnB,CAAC;yBAAM,CAAC;wBACN,oEAAoE;wBACpE,gEAAgE;wBAChE,yCAAyC;wBACzC,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBAC7D,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;4BACvB,MAAM,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;wBAC5C,CAAC;oBACH,CAAC;gBACH,CAAC;wBAAS,CAAC;oBACT,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;gBACtB,CAAC;gBACD,IAAI,SAAS;oBAAE,MAAM;YACvB,CAAC;YACD,0EAA0E;YAC1E,uEAAuE;YACvE,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3B,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,EACjD,EAAE,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,IAAI,GAAG,EAAE,YAAY,EAAE,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QACnC,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,0DAA0D;IAC1D,KAAK,CAAC,OAAO;QACX,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAwB;QACjC,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,uBAAuB,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjB,IAAI,CAAC,IAAI;gBACT,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;aACnD,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,aAAa;QACf,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;YAC5D,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC3C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC;IACxC,CAAC;CACF"}

package/dist/worker/replication-worker.d.ts

@@ -0,0 +1,170 @@
+import type { CheckpointStore } from "./checkpoint-store.js";
+import type { DeadLetterSink } from "./dead-letter.js";
+import type { DedupeStore } from "./dedupe-store.js";
+import { ObservabilityHooks } from "./observability-hooks.js";
+import { type SingleActiveLeaseOpts } from "./single-active-lease.js";
+import { type StreamsRedisLike } from "./streams-worker.js";
+import type { SqlExecutor } from "./tx.js";
+import type { WorkerHandle, WorkerHwm, WorkerStopOptions, WorkerStopResult } from "./worker-handle.js";
+/** A replication event off the source stream. Carries the LWW version. */
+export interface ReplicationEvent {
+    /** Dedupe id — `(event_id, topic)` is the composite dedupe key (D198). */
+    event_id: string;
+    /** The outbox topic. */
+    topic: string;
+    /** The replicated entity's id (the projection key). */
+    entity_id: string;
+    /** The monotonic source version — drives last-write-wins. */
+    entity_version: number | string | bigint;
+    /** Optional source emission ms for lag. */
+    occurred_at_ms?: number;
+    /** The rest of the payload (the projection input). */
+    [key: string]: unknown;
+}
+/** The apply context handed to the injected projection upsert INSIDE the tx. */
+export interface ReplicationApplyContext {
+    tx: SqlExecutor;
+    event: ReplicationEvent;
+    streamId: string;
+    /** THE fence guard — call BEFORE the projection upsert. */
+    validateFence(): Promise<void>;
+}
+/** The injected projection upsert. Runs inside the worker's apply tx. */
+export type ReplicationApplyHandler = (ctx: ReplicationApplyContext) => Promise<void>;
+/** The injected audit emitter — fires AFTER a successful commit (§ 4 audit). */
+export type ReplicationAuditHook = (info: {
+    event: ReplicationEvent;
+    streamId: string;
+    /** True when the event applied; false when LWW-skipped (still consumed). */
+    applied: boolean;
+}) => Promise<void> | void;
+/** A snapshot bootstrap source — streams the snapshot rows + the HWM to stitch. */
+export interface SnapshotBootstrap {
+    /**
+     * Apply the full snapshot into the projection and return the high-water
+     * stream id the live consume must stitch from. The worker writes this as the
+     * cursor so live consume resumes exactly there (§ 7 stitch). The bootstrap
+     * runs its own apply (it owns the projection schema); the worker only records
+     * the returned HWM as the durable cursor.
+     */
+    bootstrap(tx: SqlExecutor): Promise<{
+        highWaterStreamId: string;
+    }>;
+}
+export interface ReplicationWorkerOpts {
+    redis: StreamsRedisLike;
+    /** The source replication stream key. */
+    sourceStream: string;
+    /** The injected projection upsert. */
+    apply: ReplicationApplyHandler;
+    /**
+     * Reads the projection's CURRENT source_entity_version for an entity, for the
+     * LWW comparison. Returns null when the entity isn't projected yet (→ apply).
+     */
+    currentVersion: (tx: SqlExecutor, event: ReplicationEvent) => Promise<bigint | null>;
+    /** Opens an apply transaction; the worker runs dedupe/checkpoint/apply on it. */
+    begin: <T>(fn: (tx: SqlExecutor) => Promise<T>) => Promise<T>;
+    dedupe: DedupeStore;
+    checkpoint: CheckpointStore;
+    lease: Omit<SingleActiveLeaseOpts, "onLeaseLost">;
+    /** Checkpoint subscription key. Default `replication:<sourceStream>`. */
+    subscription?: string;
+    deadLetter: DeadLetterSink;
+    /** Optional snapshot bootstrap (run once if the cursor is absent). */
+    snapshot?: SnapshotBootstrap;
+    /** Optional audit hook (fires after each successful commit). */
+    audit?: ReplicationAuditHook;
+    retryMaxAttempts?: number;
+    pollBatch?: number;
+    blockMs?: number;
+    /** XAUTOCLAIM crash-recovery threshold (ms). Default max(lease ttlMs, 30_000). */
+    reclaimMinIdleMs?: number;
+    hooks?: ObservabilityHooks;
+    workerLabel?: string;
+    /** Launch the background consume loop on `start()`. Default true; tests pass
+     *  false to drive `runOnce()` deterministically. */
+    autoStart?: boolean;
+}
+/** Consumer-group name — `${service}:${stream}` (canonical per replica-consumer). */
+export declare function replicationConsumerGroup(service: string, stream: string): string;
+/**
+ * ReplicationWorker — the Pattern-3 subscriber. Composes lease + dedupe +
+ * checkpoint + hooks + drain + snapshot-bootstrap + LWW + audit. EXACTLY-ONCE
+ * apply, monotonic projection (no LWW regression), atomic cursor+dedupe+upsert.
+ */
+export declare class ReplicationWorker implements WorkerHandle {
+    private readonly redis;
+    private readonly sourceStream;
+    private readonly applyHandler;
+    private readonly currentVersion;
+    private readonly begin;
+    private readonly dedupe;
+    private readonly checkpoint;
+    private readonly lease;
+    private readonly subscription;
+    private readonly deadLetter;
+    private readonly snapshot?;
+    private readonly audit?;
+    private readonly retryMaxAttempts;
+    private readonly pollBatch;
+    private readonly blockMs;
+    private readonly reclaimMinIdleMs;
+    private readonly hooks;
+    private readonly autoStart;
+    private readonly group;
+    private readonly consumer;
+    private readonly retries;
+    private _running;
+    private _stopped;
+    private _inFlight;
+    private _bootstrapped;
+    private _hwm;
+    private _lastAppliedAtMs;
+    private groupEnsured;
+    private loop;
+    constructor(opts: ReplicationWorkerOpts);
+    get running(): boolean;
+    get leaseAcquired(): boolean;
+    get lag(): number | null;
+    get currentHwm(): WorkerHwm;
+    /** True once the snapshot bootstrap (if any) has run / been determined absent. */
+    get bootstrapped(): boolean;
+    private ensureGroup;
+    start(): Promise<void>;
+    /**
+     * Snapshot-bootstrap + HWM stitch (§ 7): if the durable cursor is ABSENT (cold
+     * start) AND a snapshot source is wired, run the snapshot bootstrap inside ONE
+     * tx and write the returned high-water stream id as the cursor — live consume
+     * then resumes EXACTLY from there (no gap, no double-apply of the snapshot
+     * window). If a cursor already exists we skip the snapshot and resume live.
+     */
+    maybeBootstrap(): Promise<void>;
+    private runLoop;
+    private pollOnce;
+    /**
+     * Dead-letter + XACK every structurally-unparseable (malformed-JSON) entry. A
+     * malformed entry is a POISON message (no decodable event) — leaving it in the
+     * PEL only re-cycles it through XAUTOCLAIM forever. Route the raw bytes through
+     * the (PII-redacting) dead-letter sink, then XACK so it leaves the PEL. If the
+     * sink fails, leave the entry in the PEL (recoverable; never silently dropped).
+     */
+    private handleMalformedEntries;
+    /**
+     * Apply one replication event: dedupe-skip → LWW check → apply tx (projection
+     * + dedupe.record + checkpoint.advance, fence-guarded, ONE tx) → XACK → audit.
+     */
+    private applyOne;
+    /**
+     * XACK an already-committed entry, SWALLOWING a transient ack failure. The
+     * XACK is post-commit (or post-dedupe-skip) and NOT part of the apply tx — a
+     * failing XACK is a transient redis blip, never a poison event. On failure the
+     * entry stays un-acked in the PEL and is re-acked next poll once redis
+     * recovers (the durable dedupe row makes the redelivery a no-op). Matches Go's
+     * `_, _ = Ack(...)`; a throw here must NEVER reach retry/DLQ.
+     */
+    private ackSwallow;
+    private retryOrDeadLetter;
+    runOnce(): Promise<number>;
+    stop(opts?: WorkerStopOptions): Promise<WorkerStopResult>;
+}
+//# sourceMappingURL=replication-worker.d.ts.map

package/dist/worker/replication-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replication-worker.d.ts","sourceRoot":"","sources":["../../src/worker/replication-worker.ts"],"names":[],"mappings":"AA8CA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAiB,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC7E,OAAO,EAIL,KAAK,qBAAqB,EAC3B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAKL,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EACV,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,oBAAoB,CAAC;AAE5B,0EAA0E;AAC1E,MAAM,WAAW,gBAAgB;IAC/B,0EAA0E;IAC1E,QAAQ,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACzC,2CAA2C;IAC3C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,sDAAsD;IACtD,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,gFAAgF;AAChF,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,WAAW,CAAC;IAChB,KAAK,EAAE,gBAAgB,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAChC;AAED,yEAAyE;AACzE,MAAM,MAAM,uBAAuB,GAAG,CACpC,GAAG,EAAE,uBAAuB,KACzB,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnB,gFAAgF;AAChF,MAAM,MAAM,oBAAoB,GAAG,CAAC,IAAI,EAAE;IACxC,KAAK,EAAE,gBAAgB,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,4EAA4E;IAC5E,OAAO,EAAE,OAAO,CAAC;CAClB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAE3B,mFAAmF;AACnF,MAAM,WAAW,iBAAiB;IAChC;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,EAAE,WAAW,GAAG,OAAO,CAAC;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACpE;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,gBAAgB,CAAC;IACxB,yCAAyC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,sCAAsC;IACtC,KAAK,EAAE,uBAAuB,CAAC;IAC/B;;;OAGG;IACH,cAAc,EAAE,CACd,EAAE,EAAE,WAAW,EACf,KAAK,EAAE,gBAAgB,KACpB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC5B,iFAAiF;IACjF,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,EAAE,WAAW,CAAC;IACpB,UAAU,EAAE,eAAe,CAAC;IAC5B,KAAK,EAAE,IAAI,CAAC,qBAAqB,EAAE,aAAa,CAAC,CAAC;IAClD,yEAAyE;IACzE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,cAAc,CAAC;IAC3B,sEAAsE;IACtE,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,gEAAgE;IAChE,KAAK,CAAC,EAAE,oBAAoB,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kFAAkF;IAClF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,kBAAkB,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;wDACoD;IACpD,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAYD,qFAAqF;AACrF,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,GACb,MAAM,CAER;AAED;;;;GAIG;AACH,qBAAa,iBAAkB,YAAW,YAAY;IACpD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAmB;IACzC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA0B;IACvD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAGH;IAC5B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAEN;IAChB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAkB;IAC7C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoB;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAoB;IAC9C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAElC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA6B;IACrD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,IAAI,CAGV;IACF,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,IAAI,CAAoC;gBAEpC,IAAI,EAAE,qBAAqB;IAuCvC,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,GAAG,IAAI,MAAM,GAAG,IAAI,CAGvB;IAED,IAAI,UAAU,IAAI,SAAS,CAE1B;IAED,kFAAkF;IAClF,IAAI,YAAY,IAAI,OAAO,CAE1B;YAEa,WAAW;IAiBnB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAyB5B;;;;;;OAMG;IACG,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;YA6CvB,OAAO;YAqBP,QAAQ;IAqHtB;;;;;;OAMG;YACW,sBAAsB;IAyBpC;;;OAGG;YACW,QAAQ;IA6ItB;;;;;;;OAOG;YACW,UAAU;YAQV,iBAAiB;IAwBzB,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC;IAI1B,IAAI,CAAC,IAAI,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAsBhE"}