@nodii/telemetry 0.12.0 → 0.14.0

package/dist/worker/sweeper-worker.js

@@ -0,0 +1,245 @@
+// D402 — SweeperWorker: a single-active, checkpoint-resumable INTERVAL job.
+//
+// D402 (LOCKED): "SweeperWorker — single-active, checkpoint-resumable interval
+// job." It is the SIMPLEST of the 4 worker types and the WorkerHandle reference
+// impl — every other worker type follows its lifecycle shape (acquire lease →
+// loop → bounded stop) but adds stream/outbox/replication-specific apply logic.
+//
+// ── Composition (which primitives, how) ──────────────────────────────────────
+//   - SingleActiveLease  — acquire on start; the ttl/2 renew loop runs inside
+//     the lease; `validateFence(token)` guards the sweep so a lease-lost sweeper
+//     cannot run a delete after a successor took over.
+//   - CheckpointStore    — the sweep is checkpoint-RESUMABLE: each tick advances
+//     the cursor (the injected job returns the new `lastStreamId`/position +
+//     count) inside the job's own tx, so a restart resumes from the last commit
+//     rather than re-sweeping from zero.
+//   - SignalDrain        — `stop({timeoutMs})` is registered so SIGTERM drains it.
+//   - ObservabilityHooks — emits `nodii.worker.swept_total` (+ lease/lag/hwm).
+//   - DeadLetterSink     — INJECTED (no default); a tick error is dead-lettered.
+//
+// A >grace handler is NOT double-run: single-active means only the lease holder
+// sweeps, and `validateFence` rejects a stale holder's late delete.
+import { ensurePiiRedaction } from "./dead-letter.js";
+import { ObservabilityHooks } from "./observability-hooks.js";
+import { FenceCheckUnavailableError, FenceLostError, SingleActiveLease, } from "./single-active-lease.js";
+const DEFAULT_INTERVAL_MS = 60_000;
+const DEFAULT_STOP_TIMEOUT_MS = 30_000;
+/**
+ * SweeperWorker — the reference {@link WorkerHandle}. Acquires a single-active
+ * lease, runs the injected sweep job on an interval, advances the checkpoint
+ * each tick (resumable), and drains bounded on stop.
+ */
+export class SweeperWorker {
+    job;
+    executorFactory;
+    lease;
+    checkpoint;
+    subscription;
+    intervalMs;
+    deadLetter;
+    hooks;
+    autoStart;
+    _running = false;
+    _stopped = false;
+    _inFlight = false;
+    _hwm = { lastStreamId: null };
+    _lastAppliedAtMs = null;
+    loopTimer = null;
+    constructor(opts) {
+        this.job = opts.job;
+        this.executorFactory = opts.executor;
+        this.lease = new SingleActiveLease({
+            ...opts.lease,
+            // When the renew loop detects loss, mark the lost metric — the next tick
+            // sees `leaseAcquired===false` and idles; a stale delete is fence-rejected.
+            onLeaseLost: () => this.hooks.leaseLost(),
+        });
+        this.checkpoint = opts.checkpoint;
+        this.subscription = opts.subscription ?? `sweeper:${opts.job.name}`;
+        this.intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
+        this.autoStart = opts.autoStart !== false;
+        this.deadLetter = ensurePiiRedaction(opts.deadLetter);
+        this.hooks =
+            opts.hooks ??
+                new ObservabilityHooks({
+                    attributes: {
+                        service: opts.lease.service,
+                        scope: opts.lease.scope,
+                        worker: opts.workerLabel ?? "sweeper",
+                    },
+                });
+    }
+    get running() {
+        return this._running;
+    }
+    get leaseAcquired() {
+        return this.lease.leaseAcquired;
+    }
+    get lag() {
+        if (this._lastAppliedAtMs === null)
+            return null;
+        return Math.max(0, (Date.now() - this._lastAppliedAtMs) / 1000);
+    }
+    get currentHwm() {
+        return this._hwm;
+    }
+    /** Acquire the lease + start the interval loop. Idempotent. */
+    async start() {
+        if (this._running)
+            return;
+        this._stopped = false;
+        const acquired = await this.lease.acquire();
+        if (acquired)
+            this.hooks.leaseAcquired();
+        // Hydrate the resume cursor from the durable checkpoint (cold-start → null).
+        await this.hydrateCursor();
+        this._running = true;
+        // Background interval loop. `autoStart:false` lets a test drive `runOnce()`.
+        if (this.autoStart) {
+            this.loopTimer = setInterval(() => {
+                void this.tick();
+            }, this.intervalMs);
+            this.loopTimer.unref?.();
+        }
+    }
+    /** Read the durable checkpoint so a restart resumes rather than re-sweeps. */
+    async hydrateCursor() {
+        try {
+            const ex = await this.executorFactory();
+            const cp = await this.checkpoint.get(this.subscription, ex);
+            if (cp)
+                this._hwm = { lastStreamId: cp.lastStreamId };
+        }
+        catch {
+            // A cold-start read failure is non-fatal — the cursor stays null and the
+            // first tick simply starts from the beginning of the retention horizon.
+        }
+    }
+    /** One interval tick: acquire-if-needed → runOnce → swallow into dead-letter. */
+    async tick() {
+        if (this._stopped || this._inFlight)
+            return;
+        // BUG[5]: the ENTIRE tick body — INCLUDING the autonomous re-acquire — is
+        // guarded. A transient lease-redis blip at the re-acquire moment throws out
+        // of `lease.acquire()`; if that escaped, the `void this.tick()` interval call
+        // would leak an unhandled rejection (TS) / kill the loop task (Python). It is
+        // caught here so the loop SURVIVES to the next tick, with a loud error signal.
+        try {
+            // Re-acquire if we don't hold the lease (a successor may have released it).
+            if (!this.lease.leaseAcquired) {
+                const got = await this.lease.acquire();
+                if (got)
+                    this.hooks.leaseAcquired();
+                else
+                    return; // someone else is the single-active sweeper; idle.
+            }
+            await this.runOnce();
+        }
+        catch {
+            // Either the autonomous re-acquire threw (lease-redis blip) or the
+            // dead-letter sink itself failed inside runOnce. Emit a loud error/depth
+            // signal and CONTINUE the loop — a single bad tick (or a transient
+            // lease-redis outage) must never stop retention sweeps forever.
+            this.hooks.recordDepth(-1, { error: "tick_failed" });
+        }
+    }
+    /**
+     * Run exactly ONE sweep pass. Acquires/validates the lease for the tick,
+     * invokes the injected job, advances the checkpoint, emits `swept_total`.
+     * Resolves the number of rows swept (0 when the lease is not held).
+     */
+    async runOnce() {
+        if (!this.lease.leaseAcquired) {
+            const got = await this.lease.acquire();
+            if (got)
+                this.hooks.leaseAcquired();
+            else
+                return 0;
+        }
+        this._inFlight = true;
+        const token = this.lease.fencingToken;
+        try {
+            const ex = await this.executorFactory();
+            const ctx = {
+                exec: ex,
+                lastStreamId: this._hwm.lastStreamId,
+                validateFence: async () => {
+                    if (token === null) {
+                        throw new FenceLostError({
+                            leaseKey: this.lease.key,
+                            expectedToken: -1,
+                            current: null,
+                        });
+                    }
+                    await this.lease.validateFence(token);
+                },
+            };
+            const result = await this.job.runOnce(ctx);
+            // Advance the checkpoint (resumable) when the job moved the cursor.
+            if (result.lastStreamId !== undefined && result.lastStreamId !== null) {
+                await this.checkpoint.advance({
+                    subscription: this.subscription,
+                    lastStreamId: result.lastStreamId,
+                }, ex);
+                this._hwm = { lastStreamId: result.lastStreamId };
+            }
+            this._lastAppliedAtMs = Date.now();
+            if (result.swept > 0) {
+                this.hooks.recordSwept(result.swept, {
+                    subscription: this.subscription,
+                });
+            }
+            this.hooks.recordLag(0);
+            return result.swept;
+        }
+        catch (err) {
+            // A FenceLostError means a successor took over mid-tick — DO NOT dead-letter
+            // (it's a benign lease handoff, not a poison job); just yield the tick.
+            if (err instanceof FenceLostError) {
+                this.hooks.leaseLost();
+                return 0;
+            }
+            // A transient fence-check redis failure means we could not verify the lease
+            // — skip this tick (the sweep runs again next interval). Do NOT dead-letter
+            // a transient infra blip (cross-runtime-identical).
+            if (err instanceof FenceCheckUnavailableError) {
+                this.hooks.recordDepth(-1, { error: "fence_check_unavailable" });
+                return 0;
+            }
+            // Any other terminal error → dead-letter (durable, never a silent drop).
+            await this.deadLetter.deadLetter({
+                sourceStream: this.subscription,
+                topic: this.job.name,
+                eventId: `sweep:${Date.now()}`,
+                payload: {},
+                errorMessage: err instanceof Error ? err.message : String(err),
+                retryCount: 0,
+            });
+            return 0;
+        }
+        finally {
+            this._inFlight = false;
+        }
+    }
+    /** Bounded graceful stop: halt the loop, let an in-flight tick finish, release. */
+    async stop(opts) {
+        const timeoutMs = opts?.timeoutMs ?? DEFAULT_STOP_TIMEOUT_MS;
+        const start = Date.now();
+        this._stopped = true;
+        this._running = false;
+        if (this.loopTimer !== null) {
+            clearInterval(this.loopTimer);
+            this.loopTimer = null;
+        }
+        // Wait (bounded) for an in-flight tick to flush.
+        while (this._inFlight && Date.now() - start < timeoutMs) {
+            await new Promise((r) => setTimeout(r, 10));
+        }
+        const drainedCleanly = !this._inFlight;
+        await this.lease.release();
+        const durationMs = Date.now() - start;
+        this.hooks.recordDrainDuration(durationMs);
+        return { drainedCleanly, durationMs };
+    }
+}
+//# sourceMappingURL=sweeper-worker.js.map

package/dist/worker/sweeper-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sweeper-worker.js","sourceRoot":"","sources":["../../src/worker/sweeper-worker.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,8EAA8E;AAC9E,gFAAgF;AAChF,EAAE;AACF,gFAAgF;AAChF,8EAA8E;AAC9E,iFAAiF;AACjF,uDAAuD;AACvD,iFAAiF;AACjF,6EAA6E;AAC7E,gFAAgF;AAChF,yCAAyC;AACzC,mFAAmF;AACnF,+EAA+E;AAC/E,iFAAiF;AACjF,EAAE;AACF,gFAAgF;AAChF,oEAAoE;AAEpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,0BAA0B,EAC1B,cAAc,EACd,iBAAiB,GAElB,MAAM,0BAA0B,CAAC;AAyElC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AACnC,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAEvC;;;;GAIG;AACH,MAAM,OAAO,aAAa;IACP,GAAG,CAAuB;IAC1B,eAAe,CAA2C;IAC1D,KAAK,CAAoB;IACzB,UAAU,CAAkD;IAC5D,YAAY,CAAS;IACrB,UAAU,CAAS;IACnB,UAAU,CAAiB;IAC3B,KAAK,CAAqB;IAC1B,SAAS,CAAU;IAE5B,QAAQ,GAAG,KAAK,CAAC;IACjB,QAAQ,GAAG,KAAK,CAAC;IACjB,SAAS,GAAG,KAAK,CAAC;IAClB,IAAI,GAAc,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACzC,gBAAgB,GAAkB,IAAI,CAAC;IACvC,SAAS,GAA0C,IAAI,CAAC;IAEhE,YAAY,IAAuB;QACjC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,IAAI,iBAAiB,CAAC;YACjC,GAAG,IAAI,CAAC,KAAK;YACb,yEAAyE;YACzE,4EAA4E;YAC5E,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;SAC1C,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,WAAW,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACpE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,mBAAmB,CAAC;QACzD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC;QAC1C,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,CAAC,KAAK;YACR,IAAI,CAAC,KAAK;gBACV,IAAI,kBAAkB,CAAC;oBACrB,UAAU,EAAE;wBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;wBAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;wBACvB,MAAM,EAAE,IAAI,CAAC,WAAW,IAAI,SAAS;qBACtC;iBACF,CAAC,CAAC;IACP,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;IAClC,CAAC;IAED,IAAI,GAAG;QACL,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC5C,IAAI,QAAQ;YAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;QACzC,6EAA6E;QAC7E,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,6EAA6E;QAC7E,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACnB,IAAI,CAAC,SAA+C,CAAC,KAAK,EAAE,EAAE,CAAC;QAClE,CAAC;IACH,CAAC;IAED,8EAA8E;IACtE,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YACxC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YAC5D,IAAI,EAAE;gBAAE,IAAI,CAAC,IAAI,GAAG,EAAE,YAAY,EAAE,EAAE,CAAC,YAAY,EAAE,CAAC;QACxD,CAAC;QAAC,MAAM,CAAC;YACP,yEAAyE;YACzE,wEAAwE;QAC1E,CAAC;IACH,CAAC;IAED,iFAAiF;IACzE,KAAK,CAAC,IAAI;QAChB,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5C,0EAA0E;QAC1E,4EAA4E;QAC5E,8EAA8E;QAC9E,8EAA8E;QAC9E,+EAA+E;QAC/E,IAAI,CAAC;YACH,4EAA4E;YAC5E,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;gBAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACvC,IAAI,GAAG;oBAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;;oBAC/B,OAAO,CAAC,mDAAmD;YAClE,CAAC;YACD,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,mEAAmE;YACnE,yEAAyE;YACzE,mEAAmE;YACnE,gEAAgE;YAChE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACvC,IAAI,GAAG;gBAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;;gBAC/B,OAAO,CAAC,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YACxC,MAAM,GAAG,GAAsB;gBAC7B,IAAI,EAAE,EAAE;gBACR,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY;gBACpC,aAAa,EAAE,KAAK,IAAI,EAAE;oBACxB,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;wBACnB,MAAM,IAAI,cAAc,CAAC;4BACvB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG;4BACxB,aAAa,EAAE,CAAC,CAAC;4BACjB,OAAO,EAAE,IAAI;yBACd,CAAC,CAAC;oBACL,CAAC;oBACD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;gBACxC,CAAC;aACF,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC3C,oEAAoE;YACpE,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;gBACtE,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3B;oBACE,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;iBAClC,EACD,EAAE,CACH,CAAC;gBACF,IAAI,CAAC,IAAI,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;YACpD,CAAC;YACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBACrB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE;oBACnC,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACxB,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6EAA6E;YAC7E,wEAAwE;YACxE,IAAI,GAAG,YAAY,cAAc,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBACvB,OAAO,CAAC,CAAC;YACX,CAAC;YACD,4EAA4E;YAC5E,4EAA4E;YAC5E,oDAAoD;YACpD,IAAI,GAAG,YAAY,0BAA0B,EAAE,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC,CAAC;gBACjE,OAAO,CAAC,CAAC;YACX,CAAC;YACD,yEAAyE;YACzE,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;gBACpB,OAAO,EAAE,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE;gBAC9B,OAAO,EAAE,EAAE;gBACX,YAAY,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;gBAC9D,UAAU,EAAE,CAAC;aACd,CAAC,CAAC;YACH,OAAO,CAAC,CAAC;QACX,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,mFAAmF;IACnF,KAAK,CAAC,IAAI,CAAC,IAAwB;QACjC,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,uBAAuB,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,iDAAiD;QACjD,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;YACxD,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;QACvC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC3C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC;IACxC,CAAC;CACF"}

package/dist/worker/tx.d.ts

@@ -0,0 +1,39 @@
+/** A single result row as a plain object. */
+export type SqlRow = Record<string, unknown>;
+export interface SqlResult<R extends SqlRow = SqlRow> {
+    rows: R[];
+    /** Affected-row count for write statements (INSERT/UPDATE/DELETE). */
+    rowCount: number;
+}
+/**
+ * The minimal executor a store needs: run a parameterized statement on a
+ * specific connection / transaction handle and get rows + an affected count
+ * back. Structurally compatible with node-postgres `PoolClient.query(text,
+ * values)` (which returns `{ rows, rowCount }`). A `tx` passed to a store
+ * method MUST be the SAME handle the caller runs its projection upsert on, so
+ * everything commits together-or-none.
+ */
+export interface SqlExecutor {
+    query<R extends SqlRow = SqlRow>(text: string, values?: ReadonlyArray<unknown>): Promise<SqlResult<R>>;
+}
+/**
+ * Adapt a postgres-js transaction/connection handle (the `sql` you receive
+ * inside `sql.begin(async (sql) => …)`) to {@link SqlExecutor}. postgres-js
+ * exposes `sql.unsafe(text, params)` which resolves an array-like of rows with
+ * a `.count`. The returned executor runs every statement on that exact handle,
+ * so a store write joins the caller's transaction.
+ *
+ * Usage (postgres-js):
+ *   await sql.begin(async (tx) => {
+ *     const ex = postgresJsExecutor(tx);
+ *     await applyProjection(tx, row);
+ *     await dedupe.record(eventId, topic, ex);
+ *     await checkpoint.advance(cursor, ex);
+ *   });
+ */
+export declare function postgresJsExecutor(handle: {
+    unsafe: (text: string, params?: ReadonlyArray<unknown>) => Promise<unknown> & {
+        count?: number;
+    };
+}): SqlExecutor;
+//# sourceMappingURL=tx.d.ts.map

package/dist/worker/tx.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tx.d.ts","sourceRoot":"","sources":["../../src/worker/tx.ts"],"names":[],"mappings":"AA0BA,6CAA6C;AAC7C,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAE7C,MAAM,WAAW,SAAS,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM;IAClD,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,sEAAsE;IACtE,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,EAC7B,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,GAC9B,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;CAC1B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE;IACzC,MAAM,EAAE,CACN,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,KAC5B,OAAO,CAAC,OAAO,CAAC,GAAG;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,GAAG,WAAW,CAgBd"}

package/dist/worker/tx.js

@@ -0,0 +1,51 @@
+// D402 worker substrate — the minimal SQL-executor seam shared by
+// CheckpointStore + DedupeStore + DeadLetter's table sink.
+//
+// The CORRECTNESS PROPERTY D402 pins for the checkpoint + dedupe primitives is:
+//   "advanced ATOMICALLY inside the apply transaction (cursor write + projection
+//    upsert + dedupe commit together-or-none)."
+//
+// That is only achievable if the cursor write, the dedupe record, AND the
+// caller's projection upsert all run on the SAME transaction handle, and the
+// caller is the one who BEGIN/COMMITs. So every store method here takes a `tx`
+// (an `SqlExecutor`) rather than owning a pool: the store NEVER opens its own
+// connection for a write, so it can never commit independently of the
+// projection. The caller does:
+//
+//   await pool.transaction(async (tx) => {           // pg / postgres-js tx
+//     await applyProjectionUpsert(tx, row);          // caller's projection
+//     await dedupe.record(eventId, topic, tx);       // dedupe row
+//     await checkpoint.advance({ ...cursor }, tx);    // HWM cursor
+//   });                                               // ← one COMMIT, or rollback all
+//
+// `SqlExecutor` is structurally satisfied by BOTH a node-postgres `PoolClient`
+// (the `query(text, params)` shape) AND a postgres-js tagged transaction handle
+// (which is also callable as `sql.unsafe(text, params)`). We standardize on the
+// node-postgres `query(text, values)` shape and provide a thin postgres-js
+// adapter so a postgres-js caller can pass its tx through unchanged.
+/**
+ * Adapt a postgres-js transaction/connection handle (the `sql` you receive
+ * inside `sql.begin(async (sql) => …)`) to {@link SqlExecutor}. postgres-js
+ * exposes `sql.unsafe(text, params)` which resolves an array-like of rows with
+ * a `.count`. The returned executor runs every statement on that exact handle,
+ * so a store write joins the caller's transaction.
+ *
+ * Usage (postgres-js):
+ *   await sql.begin(async (tx) => {
+ *     const ex = postgresJsExecutor(tx);
+ *     await applyProjection(tx, row);
+ *     await dedupe.record(eventId, topic, ex);
+ *     await checkpoint.advance(cursor, ex);
+ *   });
+ */
+export function postgresJsExecutor(handle) {
+    return {
+        async query(text, values = []) {
+            const res = (await handle.unsafe(text, values));
+            const rows = Array.from(res);
+            const rowCount = typeof res.count === "number" ? res.count : (res.length ?? rows.length);
+            return { rows, rowCount };
+        },
+    };
+}
+//# sourceMappingURL=tx.js.map

package/dist/worker/tx.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tx.js","sourceRoot":"","sources":["../../src/worker/tx.ts"],"names":[],"mappings":"AAAA,kEAAkE;AAClE,2DAA2D;AAC3D,EAAE;AACF,gFAAgF;AAChF,iFAAiF;AACjF,gDAAgD;AAChD,EAAE;AACF,0EAA0E;AAC1E,6EAA6E;AAC7E,+EAA+E;AAC/E,8EAA8E;AAC9E,sEAAsE;AACtE,+BAA+B;AAC/B,EAAE;AACF,4EAA4E;AAC5E,4EAA4E;AAC5E,mEAAmE;AACnE,oEAAoE;AACpE,uFAAuF;AACvF,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,gFAAgF;AAChF,2EAA2E;AAC3E,qEAAqE;AA0BrE;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAKlC;IACC,OAAO;QACL,KAAK,CAAC,KAAK,CACT,IAAY,EACZ,SAAiC,EAAE;YAEnC,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAG9B,CAAC;YACjB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,GAAmB,CAAC,CAAC;YAC7C,MAAM,QAAQ,GACZ,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1E,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/worker/worker-handle.d.ts

@@ -0,0 +1,94 @@
+/**
+ * The high-water-mark a worker has durably advanced its CheckpointStore to.
+ *
+ * `lastStreamId` is the Redis-Streams id (`<ms>-<seq>`) of the most-recent
+ * event committed inside the apply transaction (the value
+ * {@link "./checkpoint-store".CheckpointStore} persists). `null` before the
+ * first commit. `highWaterEntityVersion` mirrors the optional LWW pointer the
+ * CheckpointStore may also track (per D402's "+ optional
+ * high_water_entity_version").
+ */
+export interface WorkerHwm {
+    /** Redis-Streams id of the last event committed, or null pre-first-commit. */
+    lastStreamId: string | null;
+    /** Optional LWW entity_version high-water-mark, when the worker tracks one. */
+    highWaterEntityVersion?: bigint | null;
+}
+/**
+ * Options for {@link WorkerHandle.stop}. The drain is BOUNDED: `timeoutMs` caps
+ * how long graceful shutdown waits for in-flight work to flush + leases to
+ * release before the worker forces down. D402 spells the bound into the signal
+ * verbatim — `stop(timeoutMs)`.
+ */
+export interface WorkerStopOptions {
+    /**
+     * Max wall-clock ms to wait for in-flight work to flush + leases to release.
+     * After this elapses the worker stops forcibly (in-flight work is abandoned;
+     * the lease TTL-expires). Defaults are worker-type specific.
+     */
+    timeoutMs?: number;
+}
+/**
+ * The result of a graceful {@link WorkerHandle.stop}. `drainedCleanly` is true
+ * when all in-flight work flushed AND every held lease released within the
+ * bound; false when the timeout forced the stop with work still in flight.
+ */
+export interface WorkerStopResult {
+    drainedCleanly: boolean;
+    /** Wall-clock ms the drain actually took (≤ the requested bound + slack). */
+    durationMs: number;
+}
+/**
+ * The uniform handle every D402 worker type exposes. A service starts a worker
+ * with {@link start}, drives it (the loop is internal; {@link runOnce} is the
+ * single-tick form for tests / cron triggers), observes it via {@link running}
+ * / {@link leaseAcquired} / {@link lag} / {@link currentHwm}, and shuts it down
+ * with {@link stop} (bounded graceful drain — wired to {@link "./signal-drain"}).
+ */
+export interface WorkerHandle {
+    /**
+     * Begin the worker loop (acquire the lease if single-active, then consume /
+     * sweep / drain continuously). Idempotent: calling `start` on an
+     * already-running handle is a no-op. Resolves once the loop is running (it
+     * does NOT block until the loop finishes — that is {@link stop}).
+     */
+    start(): Promise<void>;
+    /**
+     * Stop the worker gracefully, BOUNDED by `opts.timeoutMs`: flush in-flight
+     * work, advance + commit the checkpoint, release held leases, then resolve.
+     * If the bound elapses first, force the stop and resolve with
+     * `drainedCleanly: false`. SIGTERM/SIGINT handlers call this.
+     */
+    stop(opts?: WorkerStopOptions): Promise<WorkerStopResult>;
+    /**
+     * Run exactly ONE iteration of the worker's unit of work (consume-one-batch /
+     * sweep-once / drain-one-batch) and resolve. The single-active lease (if any)
+     * is acquired/validated for the tick. Used by tests and by cron-triggered
+     * sweepers that don't want a long-lived loop. Resolves the number of items
+     * processed in the tick (0 when idle / lease not held).
+     */
+    runOnce(): Promise<number>;
+    /** Is the worker loop currently running (between start and stop)? */
+    readonly running: boolean;
+    /**
+     * Does this worker currently hold its single-active lease? Always `true` for
+     * non-leased (e.g. all-replicas) workers; reflects the live lease state for
+     * `singleActive` workers. Mirrors
+     * {@link "./single-active-lease".SingleActiveLease.leaseAcquired}.
+     */
+    readonly leaseAcquired: boolean;
+    /**
+     * Current consumer lag in SECONDS — wall-clock between the source emission of
+     * the most-recently-applied event and its apply here. `0` when caught up,
+     * `null` when nothing has been applied yet (no basis to measure). This is the
+     * value surfaced as the `nodii.worker.lag` gauge by
+     * {@link "./observability-hooks".ObservabilityHooks}.
+     */
+    readonly lag: number | null;
+    /**
+     * The durable high-water-mark the worker has advanced its checkpoint to. The
+     * value the `nodii.worker.current_hwm` gauge reflects.
+     */
+    readonly currentHwm: WorkerHwm;
+}
+//# sourceMappingURL=worker-handle.d.ts.map

package/dist/worker/worker-handle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-handle.d.ts","sourceRoot":"","sources":["../../src/worker/worker-handle.ts"],"names":[],"mappings":"AAcA;;;;;;;;;GASG;AACH,MAAM,WAAW,SAAS;IACxB,8EAA8E;IAC9E,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,+EAA+E;IAC/E,sBAAsB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,OAAO,CAAC;IACxB,6EAA6E;IAC7E,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;;OAKG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;;;;OAKG;IACH,IAAI,CAAC,IAAI,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAE1D;;;;;;OAMG;IACH,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAE3B,qEAAqE;IACrE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAEhC;;;;;;OAMG;IACH,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAE5B;;;OAGG;IACH,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC;CAChC"}

package/dist/worker/worker-handle.js

@@ -0,0 +1,15 @@
+// D402 — WorkerHandle: the uniform lifecycle interface every worker TYPE
+// (ReplicationWorker / SweeperWorker / StreamsWorker / OutboxDispatcher — the
+// next chunk) implements.
+//
+// D402 (LOCKED) pins the surface verbatim:
+//   "WorkerHandle — start / stop(timeoutMs) / runOnce / running / lag /
+//    currentHwm / leaseAcquired."
+//
+// This file is JUST the interface + the shared value types the 4 worker types
+// and the other primitives agree on. It deliberately carries NO I/O of its own
+// — a handle is a thin façade over the composed primitives (lease + checkpoint +
+// dedupe + drain + hooks). Keeping it dependency-free means a worker type can
+// implement it without pulling Redis/Postgres into the type module.
+export {};
+//# sourceMappingURL=worker-handle.js.map

package/dist/worker/worker-handle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-handle.js","sourceRoot":"","sources":["../../src/worker/worker-handle.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,8EAA8E;AAC9E,0BAA0B;AAC1B,EAAE;AACF,2CAA2C;AAC3C,wEAAwE;AACxE,kCAAkC;AAClC,EAAE;AACF,8EAA8E;AAC9E,+EAA+E;AAC/E,iFAAiF;AACjF,8EAA8E;AAC9E,oEAAoE"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nodii/telemetry",
-  "version": "0.12.0",
+  "version": "0.14.0",
   "description": "Substrate observability library for the Nodii microservice stack — OTel wrapper, log envelope, intent lifecycle, audit signal, agent registry, context propagation. Spec: planning hub docKey=telemetry.",
   "license": "MIT",
   "type": "module",
@@ -22,6 +22,14 @@
     "./outbox": {
       "types": "./dist/outbox/index.d.ts",
       "default": "./dist/outbox/index.js"
+    },
+    "./lambda": {
+      "types": "./dist/lambda/index.d.ts",
+      "default": "./dist/lambda/index.js"
+    },
+    "./worker": {
+      "types": "./dist/worker/index.d.ts",
+      "default": "./dist/worker/index.js"
     }
   },
   "files": [
@@ -82,6 +90,7 @@
     "@nodii/audit-chain": "0.10.0",
     "drizzle-orm": "^0.45.2",
     "hono": "^4.4.0",
+    "ioredis": "^5.4.0",
     "postgres": "^3.4.0",
     "typescript": "^5.9.3"
   },