@nodii/telemetry 0.12.0 → 0.14.0

package/dist/worker/replication-worker.d.ts

@@ -0,0 +1,162 @@
+import type { CheckpointStore } from "./checkpoint-store.js";
+import type { DeadLetterSink } from "./dead-letter.js";
+import type { DedupeStore } from "./dedupe-store.js";
+import { ObservabilityHooks } from "./observability-hooks.js";
+import { type SingleActiveLeaseOpts } from "./single-active-lease.js";
+import { type StreamsRedisLike } from "./streams-worker.js";
+import type { SqlExecutor } from "./tx.js";
+import type { WorkerHandle, WorkerHwm, WorkerStopOptions, WorkerStopResult } from "./worker-handle.js";
+/** A replication event off the source stream. Carries the LWW version. */
+export interface ReplicationEvent {
+    /** Dedupe id — `(event_id, topic)` is the composite dedupe key (D198). */
+    event_id: string;
+    /** The outbox topic. */
+    topic: string;
+    /** The replicated entity's id (the projection key). */
+    entity_id: string;
+    /** The monotonic source version — drives last-write-wins. */
+    entity_version: number | string | bigint;
+    /** Optional source emission ms for lag. */
+    occurred_at_ms?: number;
+    /** The rest of the payload (the projection input). */
+    [key: string]: unknown;
+}
+/** The apply context handed to the injected projection upsert INSIDE the tx. */
+export interface ReplicationApplyContext {
+    tx: SqlExecutor;
+    event: ReplicationEvent;
+    streamId: string;
+    /** THE fence guard — call BEFORE the projection upsert. */
+    validateFence(): Promise<void>;
+}
+/** The injected projection upsert. Runs inside the worker's apply tx. */
+export type ReplicationApplyHandler = (ctx: ReplicationApplyContext) => Promise<void>;
+/** The injected audit emitter — fires AFTER a successful commit (§ 4 audit). */
+export type ReplicationAuditHook = (info: {
+    event: ReplicationEvent;
+    streamId: string;
+    /** True when the event applied; false when LWW-skipped (still consumed). */
+    applied: boolean;
+}) => Promise<void> | void;
+/** A snapshot bootstrap source — streams the snapshot rows + the HWM to stitch. */
+export interface SnapshotBootstrap {
+    /**
+     * Apply the full snapshot into the projection and return the high-water
+     * stream id the live consume must stitch from. The worker writes this as the
+     * cursor so live consume resumes exactly there (§ 7 stitch). The bootstrap
+     * runs its own apply (it owns the projection schema); the worker only records
+     * the returned HWM as the durable cursor.
+     */
+    bootstrap(tx: SqlExecutor): Promise<{
+        highWaterStreamId: string;
+    }>;
+}
+export interface ReplicationWorkerOpts {
+    redis: StreamsRedisLike;
+    /** The source replication stream key. */
+    sourceStream: string;
+    /** The injected projection upsert. */
+    apply: ReplicationApplyHandler;
+    /**
+     * Reads the projection's CURRENT source_entity_version for an entity, for the
+     * LWW comparison. Returns null when the entity isn't projected yet (→ apply).
+     */
+    currentVersion: (tx: SqlExecutor, event: ReplicationEvent) => Promise<bigint | null>;
+    /** Opens an apply transaction; the worker runs dedupe/checkpoint/apply on it. */
+    begin: <T>(fn: (tx: SqlExecutor) => Promise<T>) => Promise<T>;
+    dedupe: DedupeStore;
+    checkpoint: CheckpointStore;
+    lease: Omit<SingleActiveLeaseOpts, "onLeaseLost">;
+    /** Checkpoint subscription key. Default `replication:<sourceStream>`. */
+    subscription?: string;
+    deadLetter: DeadLetterSink;
+    /** Optional snapshot bootstrap (run once if the cursor is absent). */
+    snapshot?: SnapshotBootstrap;
+    /** Optional audit hook (fires after each successful commit). */
+    audit?: ReplicationAuditHook;
+    retryMaxAttempts?: number;
+    pollBatch?: number;
+    blockMs?: number;
+    /** XAUTOCLAIM crash-recovery threshold (ms). Default max(lease ttlMs, 30_000). */
+    reclaimMinIdleMs?: number;
+    hooks?: ObservabilityHooks;
+    workerLabel?: string;
+    /** Launch the background consume loop on `start()`. Default true; tests pass
+     *  false to drive `runOnce()` deterministically. */
+    autoStart?: boolean;
+}
+/** Consumer-group name — `${service}:${stream}` (canonical per replica-consumer). */
+export declare function replicationConsumerGroup(service: string, stream: string): string;
+/**
+ * ReplicationWorker — the Pattern-3 subscriber. Composes lease + dedupe +
+ * checkpoint + hooks + drain + snapshot-bootstrap + LWW + audit. EXACTLY-ONCE
+ * apply, monotonic projection (no LWW regression), atomic cursor+dedupe+upsert.
+ */
+export declare class ReplicationWorker implements WorkerHandle {
+    private readonly redis;
+    private readonly sourceStream;
+    private readonly applyHandler;
+    private readonly currentVersion;
+    private readonly begin;
+    private readonly dedupe;
+    private readonly checkpoint;
+    private readonly lease;
+    private readonly subscription;
+    private readonly deadLetter;
+    private readonly snapshot?;
+    private readonly audit?;
+    private readonly retryMaxAttempts;
+    private readonly pollBatch;
+    private readonly blockMs;
+    private readonly reclaimMinIdleMs;
+    private readonly hooks;
+    private readonly autoStart;
+    private readonly group;
+    private readonly consumer;
+    private readonly retries;
+    private _running;
+    private _stopped;
+    private _inFlight;
+    private _bootstrapped;
+    private _hwm;
+    private _lastAppliedAtMs;
+    private groupEnsured;
+    private loop;
+    constructor(opts: ReplicationWorkerOpts);
+    get running(): boolean;
+    get leaseAcquired(): boolean;
+    get lag(): number | null;
+    get currentHwm(): WorkerHwm;
+    /** True once the snapshot bootstrap (if any) has run / been determined absent. */
+    get bootstrapped(): boolean;
+    private ensureGroup;
+    start(): Promise<void>;
+    /**
+     * Snapshot-bootstrap + HWM stitch (§ 7): if the durable cursor is ABSENT (cold
+     * start) AND a snapshot source is wired, run the snapshot bootstrap inside ONE
+     * tx and write the returned high-water stream id as the cursor — live consume
+     * then resumes EXACTLY from there (no gap, no double-apply of the snapshot
+     * window). If a cursor already exists we skip the snapshot and resume live.
+     */
+    maybeBootstrap(): Promise<void>;
+    private runLoop;
+    private pollOnce;
+    /**
+     * Apply one replication event: dedupe-skip → LWW check → apply tx (projection
+     * + dedupe.record + checkpoint.advance, fence-guarded, ONE tx) → XACK → audit.
+     */
+    private applyOne;
+    /**
+     * XACK an already-committed entry, SWALLOWING a transient ack failure. The
+     * XACK is post-commit (or post-dedupe-skip) and NOT part of the apply tx — a
+     * failing XACK is a transient redis blip, never a poison event. On failure the
+     * entry stays un-acked in the PEL and is re-acked next poll once redis
+     * recovers (the durable dedupe row makes the redelivery a no-op). Matches Go's
+     * `_, _ = Ack(...)`; a throw here must NEVER reach retry/DLQ.
+     */
+    private ackSwallow;
+    private retryOrDeadLetter;
+    runOnce(): Promise<number>;
+    stop(opts?: WorkerStopOptions): Promise<WorkerStopResult>;
+}
+//# sourceMappingURL=replication-worker.d.ts.map

package/dist/worker/replication-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replication-worker.d.ts","sourceRoot":"","sources":["../../src/worker/replication-worker.ts"],"names":[],"mappings":"AA8CA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAiB,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC7E,OAAO,EAIL,KAAK,qBAAqB,EAC3B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EACV,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,oBAAoB,CAAC;AAE5B,0EAA0E;AAC1E,MAAM,WAAW,gBAAgB;IAC/B,0EAA0E;IAC1E,QAAQ,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACzC,2CAA2C;IAC3C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,sDAAsD;IACtD,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,gFAAgF;AAChF,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,WAAW,CAAC;IAChB,KAAK,EAAE,gBAAgB,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAChC;AAED,yEAAyE;AACzE,MAAM,MAAM,uBAAuB,GAAG,CACpC,GAAG,EAAE,uBAAuB,KACzB,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnB,gFAAgF;AAChF,MAAM,MAAM,oBAAoB,GAAG,CAAC,IAAI,EAAE;IACxC,KAAK,EAAE,gBAAgB,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,4EAA4E;IAC5E,OAAO,EAAE,OAAO,CAAC;CAClB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAE3B,mFAAmF;AACnF,MAAM,WAAW,iBAAiB;IAChC;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,EAAE,WAAW,GAAG,OAAO,CAAC;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACpE;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,gBAAgB,CAAC;IACxB,yCAAyC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,sCAAsC;IACtC,KAAK,EAAE,uBAAuB,CAAC;IAC/B;;;OAGG;IACH,cAAc,EAAE,CACd,EAAE,EAAE,WAAW,EACf,KAAK,EAAE,gBAAgB,KACpB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC5B,iFAAiF;IACjF,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,EAAE,WAAW,CAAC;IACpB,UAAU,EAAE,eAAe,CAAC;IAC5B,KAAK,EAAE,IAAI,CAAC,qBAAqB,EAAE,aAAa,CAAC,CAAC;IAClD,yEAAyE;IACzE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,cAAc,CAAC;IAC3B,sEAAsE;IACtE,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,gEAAgE;IAChE,KAAK,CAAC,EAAE,oBAAoB,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kFAAkF;IAClF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,kBAAkB,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;wDACoD;IACpD,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAYD,qFAAqF;AACrF,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,GACb,MAAM,CAER;AAED;;;;GAIG;AACH,qBAAa,iBAAkB,YAAW,YAAY;IACpD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAmB;IACzC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA0B;IACvD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAGH;IAC5B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAEN;IAChB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAkB;IAC7C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoB;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAoB;IAC9C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAElC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA6B;IACrD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,IAAI,CAGV;IACF,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,IAAI,CAAoC;gBAEpC,IAAI,EAAE,qBAAqB;IAuCvC,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,GAAG,IAAI,MAAM,GAAG,IAAI,CAGvB;IAED,IAAI,UAAU,IAAI,SAAS,CAE1B;IAED,kFAAkF;IAClF,IAAI,YAAY,IAAI,OAAO,CAE1B;YAEa,WAAW;IAiBnB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAyB5B;;;;;;OAMG;IACG,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;YA6CvB,OAAO;YAqBP,QAAQ;IA+GtB;;;OAGG;YACW,QAAQ;IAkHtB;;;;;;;OAOG;YACW,UAAU;YAQV,iBAAiB;IAwBzB,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC;IAI1B,IAAI,CAAC,IAAI,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAsBhE"}

package/dist/worker/replication-worker.js

@@ -0,0 +1,513 @@
+// D402 — ReplicationWorker: the Pattern-3 SUBSCRIBER consume side — the
+// canonical `09-replication-doctrine § 4` sequence every replicating service
+// follows.
+//
+// D402 (LOCKED): "ReplicationWorker — Pattern-3, BOTH sides: producer
+// outbox-drain + subscriber consume + snapshot-bootstrap + HWM stitch + dedupe +
+// last-write-wins + audit." This file is the SUBSCRIBER consume side (the
+// producer side is OutboxDispatcher); it GENERALIZES `ts/replica-consumer/src/`
+// (which this lib does NOT edit — replica-consumer gets re-based onto this in
+// Phase 2).
+//
+// ── Composition (which primitives, how) ──────────────────────────────────────
+//   - SingleActiveLease  — single-active per (service, source_stream); ttl/2
+//     renew inside the lease. `validateFence(token)` guards the projection
+//     upsert: a lease-lost subscriber mid-apply has its stale write REJECTED.
+//   - DedupeStore        — `seen(...)` skips an already-applied (event_id, topic);
+//     `record(...)` runs INSIDE the apply tx (atomic with projection + cursor).
+//   - CheckpointStore    — the HWM cursor; `advance(...)` runs in the SAME apply
+//     tx. Also stitches snapshot→live: the bootstrap writes the snapshot HWM as
+//     the cursor, then live consume resumes from it (§ 7 stitch).
+//   - ObservabilityHooks — lease/lag/hwm/depth metrics, canonical names.
+//   - SignalDrain        — `stop({timeoutMs})` registerable; bounded drain.
+//   - DeadLetterSink     — INJECTED (no default); a poison event after retries
+//     is dead-lettered + XACK'd.
+//   - audit hook         — an injected `onApplied(...)` audit emitter fires AFTER
+//     a successful commit (the § 4 "audit" step).
+//
+// ── LAST-WRITE-WINS (the replication-specific guard) ─────────────────────────
+//   Each event carries `entity_version`. The apply is LWW: an event whose
+//   `entity_version` is <= the projection's current `source_entity_version` is a
+//   STALE replay and MUST NOT regress the projection. The worker compares
+//   against the value the injected `currentVersion(...)` returns (the consumer
+//   reads its own projection row) and SKIPS the apply (but STILL records dedupe +
+//   advances the cursor, so the stale event is consumed-once and never retried).
+//   The CheckpointStore's `high_water_entity_version` also tracks the LWW HWM
+//   monotonically (GREATEST) as a secondary guard.
+//
+// ── The consume sequence (09-replication § 4) ────────────────────────────────
+//   bootstrap (if cursor absent: snapshot → write snapshot HWM cursor) →
+//   live consume: XREADGROUP > → for each (id, event):
+//     [1] dedupe.seen? → XACK + skip
+//     [2] LWW: event.entity_version <= current? → consume-but-skip-apply
+//     [3] apply tx: projection upsert (fence-guarded) + dedupe.record +
+//         checkpoint.advance(+hwev) — ONE tx, atomic
+//     [4] XACK; audit hook; on error retry-then-dead-letter.
+import { ensurePiiRedaction } from "./dead-letter.js";
+import { hwmGaugeValue, ObservabilityHooks } from "./observability-hooks.js";
+import { FenceCheckUnavailableError, FenceLostError, SingleActiveLease, } from "./single-active-lease.js";
+import { parseAutoClaimEntries, parseStreamEntries, } from "./streams-worker.js";
+const DEFAULT_BATCH = 32;
+const DEFAULT_BLOCK_MS = 2_000;
+const DEFAULT_RETRY_MAX = 3;
+const DEFAULT_STOP_TIMEOUT_MS = 30_000;
+const DEFAULT_RECLAIM_MIN_IDLE_MS = 30_000;
+function toBigint(v) {
+    return typeof v === "bigint" ? v : BigInt(v);
+}
+/** Consumer-group name — `${service}:${stream}` (canonical per replica-consumer). */
+export function replicationConsumerGroup(service, stream) {
+    return `${service}:${stream}`;
+}
+/**
+ * ReplicationWorker — the Pattern-3 subscriber. Composes lease + dedupe +
+ * checkpoint + hooks + drain + snapshot-bootstrap + LWW + audit. EXACTLY-ONCE
+ * apply, monotonic projection (no LWW regression), atomic cursor+dedupe+upsert.
+ */
+export class ReplicationWorker {
+    redis;
+    sourceStream;
+    applyHandler;
+    currentVersion;
+    begin;
+    dedupe;
+    checkpoint;
+    lease;
+    subscription;
+    deadLetter;
+    snapshot;
+    audit;
+    retryMaxAttempts;
+    pollBatch;
+    blockMs;
+    reclaimMinIdleMs;
+    hooks;
+    autoStart;
+    group;
+    consumer;
+    retries = new Map();
+    _running = false;
+    _stopped = false;
+    _inFlight = 0;
+    _bootstrapped = false;
+    _hwm = {
+        lastStreamId: null,
+        highWaterEntityVersion: null,
+    };
+    _lastAppliedAtMs = null;
+    groupEnsured = false;
+    loop = Promise.resolve();
+    constructor(opts) {
+        this.redis = opts.redis;
+        this.sourceStream = opts.sourceStream;
+        this.applyHandler = opts.apply;
+        this.currentVersion = opts.currentVersion;
+        this.begin = opts.begin;
+        this.dedupe = opts.dedupe;
+        this.checkpoint = opts.checkpoint;
+        this.lease = new SingleActiveLease({
+            ...opts.lease,
+            onLeaseLost: () => this.hooks.leaseLost(),
+        });
+        this.subscription = opts.subscription ?? `replication:${opts.sourceStream}`;
+        this.deadLetter = ensurePiiRedaction(opts.deadLetter);
+        this.snapshot = opts.snapshot;
+        this.audit = opts.audit;
+        this.retryMaxAttempts = opts.retryMaxAttempts ?? DEFAULT_RETRY_MAX;
+        this.pollBatch = opts.pollBatch ?? DEFAULT_BATCH;
+        this.blockMs = opts.blockMs ?? DEFAULT_BLOCK_MS;
+        this.reclaimMinIdleMs =
+            opts.reclaimMinIdleMs ??
+                Math.max(opts.lease.ttlMs, DEFAULT_RECLAIM_MIN_IDLE_MS);
+        this.autoStart = opts.autoStart !== false;
+        this.hooks =
+            opts.hooks ??
+                new ObservabilityHooks({
+                    attributes: {
+                        service: opts.lease.service,
+                        scope: opts.lease.scope,
+                        worker: opts.workerLabel ?? "replication",
+                    },
+                });
+        this.group = replicationConsumerGroup(opts.lease.service, opts.sourceStream);
+        this.consumer = `${opts.lease.service}:${this.lease.ownerId}`;
+    }
+    get running() {
+        return this._running;
+    }
+    get leaseAcquired() {
+        return this.lease.leaseAcquired;
+    }
+    get lag() {
+        if (this._lastAppliedAtMs === null)
+            return null;
+        return Math.max(0, (Date.now() - this._lastAppliedAtMs) / 1000);
+    }
+    get currentHwm() {
+        return this._hwm;
+    }
+    /** True once the snapshot bootstrap (if any) has run / been determined absent. */
+    get bootstrapped() {
+        return this._bootstrapped;
+    }
+    async ensureGroup() {
+        if (this.groupEnsured)
+            return;
+        try {
+            await this.redis.xgroup("CREATE", this.sourceStream, this.group, "$", "MKSTREAM");
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (!/BUSYGROUP/i.test(msg))
+                throw err;
+        }
+        this.groupEnsured = true;
+    }
+    async start() {
+        if (this._running)
+            return;
+        this._stopped = false;
+        const acquired = await this.lease.acquire();
+        if (acquired)
+            this.hooks.leaseAcquired();
+        // ensureGroup (XGROUP CREATE) is an idempotent no-op when the group exists —
+        // safe regardless of lease ownership. The SNAPSHOT BOOTSTRAP (which WRITES the
+        // projection + cursor) runs ONLY when we hold the lease: a non-holder Start()
+        // must NOT bootstrap, or two cold instances racing into the shared projection
+        // would both snapshot (single-active violation). A non-holder idles; the loop
+        // re-acquires + bootstraps (lease-guarded) once it wins. BUG-1: release the
+        // lease on a boot-step failure after a successful acquire (never orphan it).
+        try {
+            await this.ensureGroup();
+            if (acquired)
+                await this.maybeBootstrap();
+        }
+        catch (err) {
+            if (acquired)
+                await this.lease.release();
+            throw err;
+        }
+        this._running = true;
+        // The background consume loop. A test may set `autoStart:false` to drive
+        // `runOnce()` deterministically without the loop racing the explicit ticks.
+        if (this.autoStart)
+            this.loop = this.runLoop();
+    }
+    /**
+     * Snapshot-bootstrap + HWM stitch (§ 7): if the durable cursor is ABSENT (cold
+     * start) AND a snapshot source is wired, run the snapshot bootstrap inside ONE
+     * tx and write the returned high-water stream id as the cursor — live consume
+     * then resumes EXACTLY from there (no gap, no double-apply of the snapshot
+     * window). If a cursor already exists we skip the snapshot and resume live.
+     */
+    async maybeBootstrap() {
+        if (this._bootstrapped)
+            return;
+        const existing = await this.begin((tx) => this.checkpoint.get(this.subscription, tx));
+        if (existing) {
+            // Warm start — resume live from the durable cursor.
+            this._hwm = {
+                lastStreamId: existing.lastStreamId,
+                highWaterEntityVersion: existing.highWaterEntityVersion,
+            };
+            this._bootstrapped = true;
+            return;
+        }
+        const snapshot = this.snapshot;
+        if (!snapshot) {
+            // No snapshot wired + no cursor → consume live from `>` (group at `$`).
+            this._bootstrapped = true;
+            return;
+        }
+        // Cold start WITH a snapshot — bootstrap + stitch the cursor in ONE tx so
+        // the snapshot apply + the cursor write commit together-or-none.
+        await this.begin(async (tx) => {
+            const { highWaterStreamId } = await snapshot.bootstrap(tx);
+            await this.checkpoint.advance({ subscription: this.subscription, lastStreamId: highWaterStreamId }, tx);
+        });
+        this._hwm = {
+            lastStreamId: null,
+            highWaterEntityVersion: null,
+        };
+        const cp = await this.begin((tx) => this.checkpoint.get(this.subscription, tx));
+        if (cp) {
+            this._hwm = {
+                lastStreamId: cp.lastStreamId,
+                highWaterEntityVersion: cp.highWaterEntityVersion,
+            };
+        }
+        this._bootstrapped = true;
+    }
+    async runLoop() {
+        while (!this._stopped) {
+            if (this.lease.lost)
+                break;
+            try {
+                const processed = await this.pollOnce();
+                if (processed === 0) {
+                    await new Promise((r) => setTimeout(r, 0));
+                }
+            }
+            catch (err) {
+                // BUG-2 fix: a poll that throws BECAUSE we're shutting down (stop() set
+                // `_stopped`) is a CLEAN cancel, not a redis failure — exit quietly.
+                // A genuine poll-level error (XREADGROUP rejected — Redis down) while
+                // still running is the Pillar-5 infra-failure case: back off, never crash.
+                if (this._stopped)
+                    break;
+                this.hooks.recordDepth(-1, { error: "poll_failed" });
+                void err;
+                await new Promise((r) => setTimeout(r, 200));
+            }
+        }
+    }
+    async pollOnce() {
+        let reacquiredThisPoll = false;
+        if (!this.lease.leaseAcquired) {
+            const got = await this.lease.acquire();
+            if (got) {
+                reacquiredThisPoll = true;
+                this.hooks.leaseAcquired();
+                // Re-ensure the consumer group on every re-acquire (a Redis failover may
+                // have wiped it — a cached `groupEnsured` would let xreadgroup hit
+                // NOGROUP forever).
+                this.groupEnsured = false;
+                // BUG fix (in-loop re-acquire): release the freshly-taken lease if
+                // ensureGroup throws — otherwise the renewal loop holds it INDEFINITELY
+                // while the next poll hammers a never-created group (NOGROUP loop).
+                try {
+                    await this.ensureGroup();
+                }
+                catch (err) {
+                    await this.lease.release();
+                    throw err;
+                }
+            }
+            else {
+                return 0;
+            }
+        }
+        if (!this._bootstrapped) {
+            // Same in-loop rule: a bootstrap failure right after THIS poll re-acquired
+            // must release the freshly-taken lease (a lease held from a prior poll is
+            // left intact — maybeBootstrap won't run then, `_bootstrapped` is true).
+            try {
+                await this.maybeBootstrap();
+            }
+            catch (err) {
+                if (reacquiredThisPoll)
+                    await this.lease.release();
+                throw err;
+            }
+        }
+        let processed = 0;
+        // [reclaim] CRASH-RECOVERY reaper: XAUTOCLAIM transfers PENDING entries idle
+        // ≥ reclaimMinIdleMs from a DEAD predecessor to US (the single-active
+        // successor) — without it a crashed worker's un-ACK'd entries are
+        // PERMANENTLY ORPHANED (silent loss; the "no-reaper" defect D402 eliminates).
+        // The fence guard + LWW still protect against a still-alive predecessor.
+        const claimedRaw = await this.redis.xautoclaim(this.sourceStream, this.group, this.consumer, this.reclaimMinIdleMs, "0-0", "COUNT", this.pollBatch);
+        for (const p of parseAutoClaimEntries(claimedRaw)) {
+            if (this._stopped || this.lease.lost)
+                break;
+            await this.applyOne(p.id, p.event);
+            processed += 1;
+        }
+        // [retry] Re-read our OWN PEL (id `0`) for NACK'd entries — they must be
+        // retried, and a plain `>` read never re-returns a PEL entry.
+        const pendingRaw = await this.redis.xreadgroup("GROUP", this.group, this.consumer, "COUNT", this.pollBatch, "STREAMS", this.sourceStream, "0");
+        for (const p of parseStreamEntries(pendingRaw)) {
+            if (this._stopped || this.lease.lost)
+                break;
+            await this.applyOne(p.id, p.event);
+            processed += 1;
+        }
+        // [new] Read brand-new entries (`>`, blocking).
+        const raw = await this.redis.xreadgroup("GROUP", this.group, this.consumer, "COUNT", this.pollBatch, "BLOCK", this.blockMs, "STREAMS", this.sourceStream, ">");
+        try {
+            const depth = await this.redis.xlen(this.sourceStream);
+            this.hooks.recordDepth(depth);
+        }
+        catch {
+            /* depth advisory */
+        }
+        for (const p of parseStreamEntries(raw)) {
+            if (this._stopped || this.lease.lost)
+                break;
+            await this.applyOne(p.id, p.event);
+            processed += 1;
+        }
+        return processed;
+    }
+    /**
+     * Apply one replication event: dedupe-skip → LWW check → apply tx (projection
+     * + dedupe.record + checkpoint.advance, fence-guarded, ONE tx) → XACK → audit.
+     */
+    async applyOne(streamId, event) {
+        if (this.lease.lost)
+            return;
+        this._inFlight += 1;
+        const token = this.lease.fencingToken;
+        try {
+            // [1] Pre-tx dedupe probe.
+            const alreadySeen = await this.begin((tx) => this.dedupe.seen(event.event_id, event.topic, tx));
+            if (alreadySeen) {
+                // The event already committed — only the XACK is outstanding. A failing
+                // XACK is a TRANSIENT ack-failure, NOT a poison event: swallow it (re-acked
+                // next poll). NEVER route a committed event to retry/DLQ over an ack blip.
+                await this.ackSwallow(streamId);
+                this.retries.delete(streamId);
+                return;
+            }
+            const incomingVersion = toBigint(event.entity_version);
+            let applied = false;
+            // [2]+[3] Apply tx: LWW check + projection + dedupe + cursor — ONE tx.
+            await this.begin(async (tx) => {
+                // LWW: read the projection's current version IN the tx so the compare
+                // is consistent with the upsert.
+                const current = await this.currentVersion(tx, event);
+                const isStale = current !== null && incomingVersion <= current;
+                if (!isStale) {
+                    // Apply the projection (fence-guarded BEFORE the write).
+                    const ctx = {
+                        tx,
+                        event,
+                        streamId,
+                        validateFence: async () => {
+                            if (token === null) {
+                                throw new FenceLostError({
+                                    leaseKey: this.lease.key,
+                                    expectedToken: -1,
+                                    current: null,
+                                });
+                            }
+                            await this.lease.validateFence(token);
+                        },
+                    };
+                    await this.applyHandler(ctx);
+                    applied = true;
+                }
+                // ALWAYS record dedupe + advance the cursor — even on an LWW-skip the
+                // event is consumed-once (it must never be retried) and the cursor must
+                // advance past it. The hwev advances monotonically (GREATEST) regardless.
+                await this.dedupe.record(event.event_id, event.topic, tx);
+                await this.checkpoint.advance({
+                    subscription: this.subscription,
+                    lastStreamId: streamId,
+                    highWaterEntityVersion: incomingVersion,
+                }, tx);
+            });
+            // [4] Commit succeeded → XACK + audit + advance local HWM. The XACK is
+            //     POST-COMMIT and NOT part of the apply tx: a failing XACK is a
+            //     transient ack-failure, NOT a poison event — swallow it (re-acked next
+            //     poll; the dedupe row makes a redelivery a no-op). Do NOT route a
+            //     successfully-committed event to retry/DLQ over an ack blip.
+            await this.ackSwallow(streamId);
+            this.retries.delete(streamId);
+            const prevHwev = this._hwm.highWaterEntityVersion ?? null;
+            this._hwm = {
+                lastStreamId: streamId,
+                highWaterEntityVersion: prevHwev !== null && prevHwev > incomingVersion
+                    ? prevHwev
+                    : incomingVersion,
+            };
+            this._lastAppliedAtMs = Date.now();
+            this.hooks.recordHwm(hwmGaugeValue(streamId));
+            if (event.occurred_at_ms !== undefined) {
+                this.hooks.recordLag(Math.max(0, (Date.now() - event.occurred_at_ms) / 1000));
+            }
+            else {
+                this.hooks.recordLag(0);
+            }
+            if (this.audit) {
+                try {
+                    await this.audit({ event, streamId, applied });
+                }
+                catch {
+                    // The audit hook is best-effort AFTER commit — a failure must not
+                    // un-apply the projection (already committed); log-and-continue.
+                }
+            }
+        }
+        catch (err) {
+            if (err instanceof FenceLostError) {
+                this.hooks.leaseLost();
+                return;
+            }
+            // Transient fence-check redis failure: fail closed, leave the event in the
+            // PEL (no XACK, no DLQ) to reprocess on recovery — cross-runtime-identical.
+            if (err instanceof FenceCheckUnavailableError) {
+                this.hooks.recordDepth(-1, { error: "fence_check_unavailable" });
+                return;
+            }
+            await this.retryOrDeadLetter(streamId, event, err);
+        }
+        finally {
+            this._inFlight -= 1;
+        }
+    }
+    /**
+     * XACK an already-committed entry, SWALLOWING a transient ack failure. The
+     * XACK is post-commit (or post-dedupe-skip) and NOT part of the apply tx — a
+     * failing XACK is a transient redis blip, never a poison event. On failure the
+     * entry stays un-acked in the PEL and is re-acked next poll once redis
+     * recovers (the durable dedupe row makes the redelivery a no-op). Matches Go's
+     * `_, _ = Ack(...)`; a throw here must NEVER reach retry/DLQ.
+     */
+    async ackSwallow(streamId) {
+        try {
+            await this.redis.xack(this.sourceStream, this.group, streamId);
+        }
+        catch {
+            this.hooks.recordDepth(-1, { error: "ack_failed" });
+        }
+    }
+    async retryOrDeadLetter(streamId, event, err) {
+        const attempt = (this.retries.get(streamId) ?? 0) + 1;
+        this.retries.set(streamId, attempt);
+        if (attempt < this.retryMaxAttempts)
+            return; // stays in the PEL, redelivered.
+        try {
+            await this.deadLetter.deadLetter({
+                sourceStream: this.sourceStream,
+                topic: event.topic,
+                eventId: event.event_id,
+                payload: event,
+                errorMessage: err instanceof Error ? err.message : String(err),
+                retryCount: attempt,
+            });
+            await this.redis.xack(this.sourceStream, this.group, streamId);
+            this.retries.delete(streamId);
+        }
+        catch {
+            // Dead-letter sink failed — leave in the PEL rather than lose it.
+        }
+    }
+    async runOnce() {
+        return this.pollOnce();
+    }
+    async stop(opts) {
+        const timeoutMs = opts?.timeoutMs ?? DEFAULT_STOP_TIMEOUT_MS;
+        const start = Date.now();
+        this._stopped = true;
+        this._running = false;
+        try {
+            await Promise.race([
+                this.loop,
+                new Promise((r) => setTimeout(r, timeoutMs)),
+            ]);
+        }
+        catch {
+            /* swallow */
+        }
+        while (this._inFlight > 0 && Date.now() - start < timeoutMs) {
+            await new Promise((r) => setTimeout(r, 10));
+        }
+        const drainedCleanly = this._inFlight === 0;
+        await this.lease.release();
+        const durationMs = Date.now() - start;
+        this.hooks.recordDrainDuration(durationMs);
+        return { drainedCleanly, durationMs };
+    }
+}
+//# sourceMappingURL=replication-worker.js.map