@nodii/telemetry 0.13.0 → 0.14.0

package/dist/worker/streams-worker.js

@@ -0,0 +1,488 @@
+// D402 — StreamsWorker: the generic (non-replication) Redis-Streams consumer —
+// the canonical Pattern-2 begin/completion-event consumer.
+//
+// D402 (LOCKED): "StreamsWorker — generic (non-replication) stream consumer; the
+// canonical Pattern-2 begin/completion-event consumer." It closes the gap D402
+// names directly: "the Pattern-2 begin/completion-event consumer must be
+// StreamsWorker (today hand-rolled per service via
+// idempotency.wrapConsumerWorker)."
+//
+// ── Composition (which primitives, how) ──────────────────────────────────────
+//   - SingleActiveLease  — single-active per (service, stream); ttl/2 renew loop
+//     runs inside it. `validateFence(token)` guards the apply side-effect: a
+//     lease-lost worker mid-apply has its stale write REJECTED (the key test).
+//   - DedupeStore        — `seen(...)` skips an already-applied (event_id, topic)
+//     before any work; `record(...)` runs INSIDE the apply tx so the dedupe row
+//     commits together-or-none with the projection + the cursor advance.
+//   - CheckpointStore    — `advance(...)` runs in the SAME apply tx → HWM cursor
+//     never advances past an un-applied event.
+//   - ObservabilityHooks — lease/lag/hwm/depth metrics, canonical names.
+//   - SignalDrain        — `stop({timeoutMs})` is registerable; bounded drain.
+//   - DeadLetterSink     — INJECTED (no default); a poison event after retries
+//     is dead-lettered + XACK'd so the PEL doesn't grow unbounded.
+//
+// ── The consume sequence (per 01-comms § 9.2 / 09-replication § 4) ───────────
+//   XREADGROUP > → for each (id, event):
+//     [1] dedupe.seen? → XACK + skip (already applied; redelivery is a no-op)
+//     [2] apply tx: projection upsert + dedupe.record + checkpoint.advance, with
+//         validateFence(token) called BEFORE the side-effect (the fence guard)
+//     [3] XACK on success; on error retry-then-dead-letter.
+//
+// The apply itself is the INJECTED handler — the caller owns the projection
+// upsert (this lib doesn't know the consumer's schema). The handler receives the
+// tx executor + the dedupe/checkpoint primitives pre-bound so it runs everything
+// on ONE transaction.
+import { ensurePiiRedaction } from "./dead-letter.js";
+import { ObservabilityHooks } from "./observability-hooks.js";
+import { FenceCheckUnavailableError, FenceLostError, SingleActiveLease, } from "./single-active-lease.js";
+const DEFAULT_BATCH = 32;
+const DEFAULT_BLOCK_MS = 2_000;
+const DEFAULT_RETRY_MAX = 3;
+const DEFAULT_STOP_TIMEOUT_MS = 30_000;
+const DEFAULT_RECLAIM_MIN_IDLE_MS = 30_000;
+/** Consumer-group name — `${service}:${stream}` (canonical per replica-consumer). */
+export function streamsConsumerGroup(service, stream) {
+    return `${service}:${stream}`;
+}
+/** Parse an ioredis XREADGROUP response into flat (id, event) pairs. */
+export function parseStreamEntries(raw) {
+    if (!Array.isArray(raw))
+        return [];
+    const out = [];
+    for (const streamEntry of raw) {
+        if (!Array.isArray(streamEntry) || streamEntry.length < 2)
+            continue;
+        const entries = streamEntry[1];
+        if (!Array.isArray(entries))
+            continue;
+        for (const e of entries) {
+            if (!Array.isArray(e) || e.length < 2)
+                continue;
+            const id = String(e[0]);
+            const fields = e[1];
+            if (!Array.isArray(fields))
+                continue;
+            let payload;
+            for (let i = 0; i < fields.length - 1; i += 2) {
+                if (String(fields[i]) === "event") {
+                    payload = String(fields[i + 1]);
+                    break;
+                }
+            }
+            if (payload === undefined)
+                continue;
+            try {
+                out.push({ id, event: JSON.parse(payload) });
+            }
+            catch {
+                // Malformed payload — skip (the entry stays in the PEL; an operator can
+                // inspect it). Never crash the parse loop on one bad entry.
+            }
+        }
+    }
+    return out;
+}
+/**
+ * Parse an ioredis XAUTOCLAIM reply `[cursor, entries, deleted]` into flat
+ * (id, event) pairs. `entries` has the same `[[id, [f, v, ...]], ...]` shape as
+ * one stream's entries in an XREADGROUP reply, so we wrap it and reuse
+ * {@link parseStreamEntries}.
+ */
+export function parseAutoClaimEntries(raw) {
+    if (!Array.isArray(raw) || raw.length < 2)
+        return [];
+    const entries = raw[1];
+    if (!Array.isArray(entries))
+        return [];
+    // Wrap as a single-stream XREADGROUP shape: [[<stream>, <entries>]].
+    return parseStreamEntries([["", entries]]);
+}
+/**
+ * StreamsWorker — the generic Pattern-2 single-active stream consumer. Composes
+ * lease + dedupe + checkpoint + hooks + drain to consume EXACTLY-ONCE.
+ */
+export class StreamsWorker {
+    redis;
+    stream;
+    apply;
+    begin;
+    dedupe;
+    checkpoint;
+    lease;
+    subscription;
+    deadLetter;
+    retryMaxAttempts;
+    pollBatch;
+    blockMs;
+    reclaimMinIdleMs;
+    hooks;
+    autoStart;
+    group;
+    consumer;
+    retries = new Map();
+    _running = false;
+    _stopped = false;
+    _inFlight = 0;
+    _hwm = { lastStreamId: null };
+    _lastAppliedAtMs = null;
+    groupEnsured = false;
+    loop = Promise.resolve();
+    constructor(opts) {
+        this.redis = opts.redis;
+        this.stream = opts.stream;
+        this.apply = opts.apply;
+        this.begin = opts.begin;
+        this.dedupe = opts.dedupe;
+        this.checkpoint = opts.checkpoint;
+        this.lease = new SingleActiveLease({
+            ...opts.lease,
+            onLeaseLost: () => this.hooks.leaseLost(),
+        });
+        this.subscription = opts.subscription ?? `streams:${opts.stream}`;
+        this.deadLetter = ensurePiiRedaction(opts.deadLetter);
+        this.retryMaxAttempts = opts.retryMaxAttempts ?? DEFAULT_RETRY_MAX;
+        this.pollBatch = opts.pollBatch ?? DEFAULT_BATCH;
+        this.blockMs = opts.blockMs ?? DEFAULT_BLOCK_MS;
+        this.reclaimMinIdleMs =
+            opts.reclaimMinIdleMs ??
+                Math.max(opts.lease.ttlMs, DEFAULT_RECLAIM_MIN_IDLE_MS);
+        this.autoStart = opts.autoStart !== false;
+        this.hooks =
+            opts.hooks ??
+                new ObservabilityHooks({
+                    attributes: {
+                        service: opts.lease.service,
+                        scope: opts.lease.scope,
+                        worker: opts.workerLabel ?? "streams",
+                    },
+                });
+        this.group = streamsConsumerGroup(opts.lease.service, opts.stream);
+        this.consumer = `${opts.lease.service}:${this.lease.ownerId}`;
+    }
+    get running() {
+        return this._running;
+    }
+    get leaseAcquired() {
+        return this.lease.leaseAcquired;
+    }
+    get lag() {
+        if (this._lastAppliedAtMs === null)
+            return null;
+        return Math.max(0, (Date.now() - this._lastAppliedAtMs) / 1000);
+    }
+    get currentHwm() {
+        return this._hwm;
+    }
+    /** Idempotently create the consumer group (swallow BUSYGROUP). */
+    async ensureGroup() {
+        if (this.groupEnsured)
+            return;
+        try {
+            await this.redis.xgroup("CREATE", this.stream, this.group, "$", "MKSTREAM");
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (!/BUSYGROUP/i.test(msg))
+                throw err;
+        }
+        this.groupEnsured = true;
+    }
+    async start() {
+        if (this._running)
+            return;
+        this._stopped = false;
+        const acquired = await this.lease.acquire();
+        if (acquired)
+            this.hooks.leaseAcquired();
+        // BUG-1 fix: if ANY boot step AFTER a successful acquire fails (ensureGroup
+        // throws — redis down at boot / a non-BUSYGROUP error), RELEASE the lease
+        // before rethrowing. A boot failure must NOT orphan the lease (a held key
+        // blocks a restart until the TTL lapses).
+        try {
+            await this.ensureGroup();
+            await this.hydrateCursor();
+        }
+        catch (err) {
+            if (acquired)
+                await this.lease.release();
+            throw err;
+        }
+        this._running = true;
+        // Background consume loop. `autoStart:false` lets a test drive `runOnce()`.
+        if (this.autoStart)
+            this.loop = this.runLoop();
+    }
+    async hydrateCursor() {
+        try {
+            await this.begin(async (tx) => {
+                const cp = await this.checkpoint.get(this.subscription, tx);
+                if (cp)
+                    this._hwm = { lastStreamId: cp.lastStreamId };
+            });
+        }
+        catch {
+            // Cold-start read failure — cursor stays null, consume starts from `>`.
+        }
+    }
+    async runLoop() {
+        while (!this._stopped) {
+            if (this.lease.lost)
+                break;
+            try {
+                const processed = await this.pollOnce();
+                if (processed === 0) {
+                    // Yield so a synchronous test-double doesn't tight-loop.
+                    await new Promise((r) => setTimeout(r, 0));
+                }
+            }
+            catch (err) {
+                // BUG-2 fix: a poll that throws BECAUSE we're shutting down (stop() set
+                // `_stopped` and may have torn down the redis client mid-poll) is a CLEAN
+                // cancel, NOT a redis failure — exit quietly (no error metric, no
+                // error-sleep). A genuine poll-level error (XREADGROUP rejected — Redis
+                // down) while still running is the Pillar-5 infra-failure case: log via
+                // metric + back off, NEVER crash the loop.
+                if (this._stopped)
+                    break;
+                this.hooks.recordDepth(-1, { error: "poll_failed" });
+                void err;
+                await new Promise((r) => setTimeout(r, 200));
+            }
+        }
+    }
+    /**
+     * Poll one batch + apply each entry. Returns count processed.
+     *
+     * Two reads per poll: FIRST our OWN pending-entries-list (PEL) at id `0` —
+     * entries delivered to this consumer but NOT yet XACK'd (a prior apply failed
+     * + NACK'd them); they MUST be retried, and a plain `>` read never re-returns
+     * a PEL entry. THEN `>` for brand-new entries. Without the PEL re-read a
+     * poison event would be applied exactly once and never retried.
+     */
+    async pollOnce() {
+        if (!this.lease.leaseAcquired) {
+            const got = await this.lease.acquire();
+            if (got) {
+                this.hooks.leaseAcquired();
+                // Re-ensure the consumer group on every re-acquire: a Redis failover
+                // across the prior holder's death may have wiped the group, so a cached
+                // `groupEnsured` would let xreadgroup hit NOGROUP forever. Re-running is a
+                // harmless BUSYGROUP no-op when the group still exists.
+                this.groupEnsured = false;
+                // BUG fix (in-loop re-acquire): if ensureGroup throws AFTER this poll
+                // re-acquired the lease, RELEASE it before rethrowing — otherwise the
+                // renewal loop holds + renews the lease INDEFINITELY while the next poll
+                // skips this block (leaseAcquired is true) and hammers a never-created
+                // group (NOGROUP loop), deadlocking every other instance.
+                try {
+                    await this.ensureGroup();
+                }
+                catch (err) {
+                    await this.lease.release();
+                    throw err;
+                }
+            }
+            else {
+                return 0;
+            }
+        }
+        let processed = 0;
+        // [reclaim] CRASH-RECOVERY reaper: XAUTOCLAIM transfers PENDING entries idle
+        // ≥ reclaimMinIdleMs from a DEAD predecessor (a crashed worker with a
+        // different consumer name) to US, the single-active successor. Without this,
+        // a crashed worker's un-ACK'd in-flight entries are PERMANENTLY ORPHANED (a
+        // plain "0" PEL re-read is per-consumer) → silent event loss → broken
+        // at-least-once. The fence guard inside the apply tx still rejects a stale
+        // write if a still-alive predecessor races us.
+        const claimedRaw = await this.redis.xautoclaim(this.stream, this.group, this.consumer, this.reclaimMinIdleMs, "0-0", "COUNT", this.pollBatch);
+        for (const p of parseAutoClaimEntries(claimedRaw)) {
+            if (this._stopped || this.lease.lost)
+                break;
+            await this.applyOne(p.id, p.event);
+            processed += 1;
+        }
+        // [retry] Re-read our own PEL (id `0`, non-blocking) for NACK'd entries.
+        const pendingRaw = await this.redis.xreadgroup("GROUP", this.group, this.consumer, "COUNT", this.pollBatch, "STREAMS", this.stream, "0");
+        for (const p of parseStreamEntries(pendingRaw)) {
+            if (this._stopped || this.lease.lost)
+                break;
+            await this.applyOne(p.id, p.event);
+            processed += 1;
+        }
+        // [new] Read brand-new entries (`>`, blocking up to blockMs).
+        const raw = await this.redis.xreadgroup("GROUP", this.group, this.consumer, "COUNT", this.pollBatch, "BLOCK", this.blockMs, "STREAMS", this.stream, ">");
+        // Sample backlog depth (cheap; best-effort).
+        try {
+            const depth = await this.redis.xlen(this.stream);
+            this.hooks.recordDepth(depth);
+        }
+        catch {
+            /* depth is advisory — never fail the poll on it */
+        }
+        for (const p of parseStreamEntries(raw)) {
+            if (this._stopped || this.lease.lost)
+                break;
+            await this.applyOne(p.id, p.event);
+            processed += 1;
+        }
+        return processed;
+    }
+    /**
+     * Apply one entry: dedupe-skip → apply-tx (projection + dedupe.record +
+     * checkpoint.advance, fence-guarded) → XACK. On error, retry-then-dead-letter.
+     */
+    async applyOne(streamId, event) {
+        // We dropped the lease mid-flight; the entry stays un-ACK'd in our PEL. The
+        // single-active successor (a new instance with a DIFFERENT consumer name)
+        // reclaims it via the [reclaim] XAUTOCLAIM reaper once idle — NOT via a plain
+        // "0" PEL re-read (that is per-consumer).
+        if (this.lease.lost)
+            return;
+        this._inFlight += 1;
+        const token = this.lease.fencingToken;
+        try {
+            // [1] Pre-tx dedupe probe (cheap). A redelivered already-applied event is
+            //     XACK'd + skipped (exactly-once on the consumer side).
+            const alreadySeen = await this.begin((tx) => this.dedupe.seen(event.event_id, event.topic, tx));
+            if (alreadySeen) {
+                // The event already committed (durable dedupe row) — only the XACK is
+                // outstanding. A failing XACK here is a TRANSIENT ack-failure, NOT a
+                // poison event: swallow it (the entry stays un-acked in the PEL, re-acked
+                // next poll once redis recovers). NEVER route a committed event to
+                // retry/DLQ over an ack blip (matches Go's `_, _ = Ack(...)`).
+                await this.ackSwallow(streamId);
+                this.retries.delete(streamId);
+                return;
+            }
+            // [2] Apply tx — projection upsert + dedupe.record + checkpoint.advance,
+            //     all on ONE tx (atomic). The fence guard runs BEFORE the projection.
+            await this.begin(async (tx) => {
+                const ctx = {
+                    tx,
+                    event,
+                    streamId,
+                    validateFence: async () => {
+                        if (token === null) {
+                            throw new FenceLostError({
+                                leaseKey: this.lease.key,
+                                expectedToken: -1,
+                                current: null,
+                            });
+                        }
+                        await this.lease.validateFence(token);
+                    },
+                };
+                await this.apply(ctx);
+                await this.dedupe.record(event.event_id, event.topic, tx);
+                await this.checkpoint.advance({ subscription: this.subscription, lastStreamId: streamId }, tx);
+            });
+            // [3] Commit succeeded → XACK + advance local HWM. The XACK is POST-COMMIT
+            //     and NOT part of the apply tx: a failing XACK here is a transient
+            //     ack-failure, NOT a poison event — swallow it (the committed event is
+            //     re-acked next poll; the dedupe row makes a redelivery a no-op). Do NOT
+            //     route a successfully-committed event to retry/DLQ over an ack blip.
+            await this.ackSwallow(streamId);
+            this.retries.delete(streamId);
+            this._hwm = { lastStreamId: streamId };
+            this._lastAppliedAtMs = Date.now();
+            if (event.occurred_at_ms !== undefined) {
+                this.hooks.recordLag(Math.max(0, (Date.now() - event.occurred_at_ms) / 1000));
+            }
+            else {
+                this.hooks.recordLag(0);
+            }
+        }
+        catch (err) {
+            // A FenceLostError means the lease was lost/taken-over mid-apply — the
+            // stale write was REJECTED before it landed. Do NOT XACK (the entry stays
+            // in our PEL; the single-active successor reclaims it via the XAUTOCLAIM
+            // reaper once idle) and do NOT dead-letter (benign handoff).
+            if (err instanceof FenceLostError) {
+                this.hooks.leaseLost();
+                return;
+            }
+            // A FenceCheckUnavailableError means the fence-check READ failed (redis
+            // down) — TRANSIENT. Fail closed (the write was withheld), but do NOT XACK
+            // and do NOT dead-letter a non-poison event over a redis blip: leave it in
+            // the PEL to be reprocessed once redis recovers (cross-runtime-identical).
+            if (err instanceof FenceCheckUnavailableError) {
+                this.hooks.recordDepth(-1, { error: "fence_check_unavailable" });
+                return;
+            }
+            await this.retryOrDeadLetter(streamId, event, err);
+        }
+        finally {
+            this._inFlight -= 1;
+        }
+    }
+    /**
+     * XACK an already-committed entry, SWALLOWING a transient ack failure. The
+     * XACK is post-commit (or post-dedupe-skip) and NOT part of the apply tx — a
+     * failing XACK is a transient redis blip, never a poison event. On failure the
+     * entry simply stays un-acked in the PEL and is re-acked next poll once redis
+     * recovers (the durable dedupe row makes the redelivery a no-op). This is what
+     * Go does (`_, _ = Ack(...)`); a throw here must NEVER reach retry/DLQ.
+     */
+    async ackSwallow(streamId) {
+        try {
+            await this.redis.xack(this.stream, this.group, streamId);
+        }
+        catch {
+            this.hooks.recordDepth(-1, { error: "ack_failed" });
+        }
+    }
+    /** Increment the retry ledger; dead-letter (+XACK) once the budget is spent. */
+    async retryOrDeadLetter(streamId, event, err) {
+        const attempt = (this.retries.get(streamId) ?? 0) + 1;
+        this.retries.set(streamId, attempt);
+        if (attempt < this.retryMaxAttempts) {
+            // No XACK — the entry stays in the PEL and is redelivered next poll.
+            return;
+        }
+        try {
+            await this.deadLetter.deadLetter({
+                sourceStream: this.stream,
+                topic: event.topic,
+                eventId: event.event_id,
+                payload: event,
+                errorMessage: err instanceof Error ? err.message : String(err),
+                retryCount: attempt,
+            });
+            // ACK after the dead-letter is durable so the PEL doesn't grow unbounded.
+            await this.redis.xack(this.stream, this.group, streamId);
+            this.retries.delete(streamId);
+        }
+        catch {
+            // The dead-letter sink itself failed — leave the entry in the PEL (it will
+            // be redelivered) rather than losing it. Never XACK on a failed dead-letter.
+        }
+    }
+    /** Run exactly one poll batch (test seam / cron tick). */
+    async runOnce() {
+        return this.pollOnce();
+    }
+    async stop(opts) {
+        const timeoutMs = opts?.timeoutMs ?? DEFAULT_STOP_TIMEOUT_MS;
+        const start = Date.now();
+        this._stopped = true;
+        this._running = false;
+        try {
+            await Promise.race([
+                this.loop,
+                new Promise((r) => setTimeout(r, timeoutMs)),
+            ]);
+        }
+        catch {
+            /* swallow */
+        }
+        while (this._inFlight > 0 && Date.now() - start < timeoutMs) {
+            await new Promise((r) => setTimeout(r, 10));
+        }
+        const drainedCleanly = this._inFlight === 0;
+        await this.lease.release();
+        const durationMs = Date.now() - start;
+        this.hooks.recordDrainDuration(durationMs);
+        return { drainedCleanly, durationMs };
+    }
+}
+//# sourceMappingURL=streams-worker.js.map

package/dist/worker/streams-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"streams-worker.js","sourceRoot":"","sources":["../../src/worker/streams-worker.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,2DAA2D;AAC3D,EAAE;AACF,iFAAiF;AACjF,+EAA+E;AAC/E,yEAAyE;AACzE,mDAAmD;AACnD,oCAAoC;AACpC,EAAE;AACF,gFAAgF;AAChF,iFAAiF;AACjF,6EAA6E;AAC7E,+EAA+E;AAC/E,kFAAkF;AAClF,gFAAgF;AAChF,yEAAyE;AACzE,iFAAiF;AACjF,+CAA+C;AAC/C,yEAAyE;AACzE,+EAA+E;AAC/E,+EAA+E;AAC/E,mEAAmE;AACnE,EAAE;AACF,gFAAgF;AAChF,yCAAyC;AACzC,8EAA8E;AAC9E,iFAAiF;AACjF,+EAA+E;AAC/E,4DAA4D;AAC5D,EAAE;AACF,4EAA4E;AAC5E,iFAAiF;AACjF,iFAAiF;AACjF,sBAAsB;AAGtB,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAGtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,0BAA0B,EAC1B,cAAc,EACd,iBAAiB,GAElB,MAAM,0BAA0B,CAAC;AA+GlC,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAC/B,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,uBAAuB,GAAG,MAAM,CAAC;AACvC,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAE3C,qFAAqF;AACrF,MAAM,UAAU,oBAAoB,CAAC,OAAe,EAAE,MAAc;IAClE,OAAO,GAAG,OAAO,IAAI,MAAM,EAAE,CAAC;AAChC,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,kBAAkB,CAChC,GAAY;IAEZ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,GAAG,GAA8C,EAAE,CAAC;IAC1D,KAAK,MAAM,WAAW,IAAI,GAAgB,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QACpE,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,SAAS;QACtC,KAAK,MAAM,CAAC,IAAI,OAAoB,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAChD,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,SAAS;YACrC,IAAI,OAA2B,CAAC;YAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;oBAClC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,OAAO,KAAK,SAAS;gBAAE,SAAS;YACpC,IAAI,CAAC;gBACH,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAgB,EAAE,CAAC,CAAC;YAC9D,CAAC;YAAC,MAAM,CAAC;gBACP,wEAAwE;gBACxE,4DAA4D;YAC9D,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CACnC,GAAY;IAEZ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACrD,MAAM,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACvB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,qEAAqE;IACrE,OAAO,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,aAAa;IACP,KAAK,CAAmB;IACxB,MAAM,CAAS;IACf,KAAK,CAAqB;IAC1B,KAAK,CAEN;IACC,MAAM,CAAc;IACpB,UAAU,CAAkB;IAC5B,KAAK,CAAoB;IACzB,YAAY,CAAS;IACrB,UAAU,CAAiB;IAC3B,gBAAgB,CAAS;IACzB,SAAS,CAAS;IAClB,OAAO,CAAS;IAChB,gBAAgB,CAAS;IACzB,KAAK,CAAqB;IAC1B,SAAS,CAAU;IACnB,KAAK,CAAS;IACd,QAAQ,CAAS;IAEjB,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,QAAQ,GAAG,KAAK,CAAC;IACjB,QAAQ,GAAG,KAAK,CAAC;IACjB,SAAS,GAAG,CAAC,CAAC;IACd,IAAI,GAAc,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACzC,gBAAgB,GAAkB,IAAI,CAAC;IACvC,YAAY,GAAG,KAAK,CAAC;IACrB,IAAI,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEhD,YAAY,IAAuB;QACjC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,IAAI,iBAAiB,CAAC;YACjC,GAAG,IAAI,CAAC,KAAK;YACb,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;SAC1C,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE,CAAC;QAClE,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,aAAa,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,gBAAgB,CAAC;QAChD,IAAI,CAAC,gBAAgB;YACnB,IAAI,CAAC,gBAAgB;gBACrB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC;QAC1D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC;QAC1C,IAAI,CAAC,KAAK;YACR,IAAI,CAAC,KAAK;gBACV,IAAI,kBAAkB,CAAC;oBACrB,UAAU,EAAE;wBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;wBAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;wBACvB,MAAM,EAAE,IAAI,CAAC,WAAW,IAAI,SAAS;qBACtC;iBACF,CAAC,CAAC;QACL,IAAI,CAAC,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACnE,IAAI,CAAC,QAAQ,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;IAChE,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;IAClC,CAAC;IAED,IAAI,GAAG;QACL,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,kEAAkE;IAC1D,KAAK,CAAC,WAAW;QACvB,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QAC9B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CACrB,QAAQ,EACR,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,GAAG,EACH,UAAU,CACX,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,MAAM,GAAG,CAAC;QACzC,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC5C,IAAI,QAAQ;YAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;QACzC,4EAA4E;QAC5E,0EAA0E;QAC1E,0EAA0E;QAC1E,0CAA0C;QAC1C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,QAAQ;gBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACzC,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,4EAA4E;QAC5E,IAAI,IAAI,CAAC,SAAS;YAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAC5B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAC5D,IAAI,EAAE;oBAAE,IAAI,CAAC,IAAI,GAAG,EAAE,YAAY,EAAE,EAAE,CAAC,YAAY,EAAE,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,wEAAwE;QAC1E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC3B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACxC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;oBACpB,yDAAyD;oBACzD,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,wEAAwE;gBACxE,0EAA0E;gBAC1E,kEAAkE;gBAClE,wEAAwE;gBACxE,wEAAwE;gBACxE,2CAA2C;gBAC3C,IAAI,IAAI,CAAC,QAAQ;oBAAE,MAAM;gBACzB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;gBACrD,KAAK,GAAG,CAAC;gBACT,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,QAAQ;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACvC,IAAI,GAAG,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;gBAC3B,qEAAqE;gBACrE,wEAAwE;gBACxE,2EAA2E;gBAC3E,wDAAwD;gBACxD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;gBAC1B,sEAAsE;gBACtE,sEAAsE;gBACtE,yEAAyE;gBACzE,uEAAuE;gBACvE,0DAA0D;gBAC1D,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC3B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;oBAC3B,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,CAAC;YACX,CAAC;QACH,CAAC;QACD,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,6EAA6E;QAC7E,sEAAsE;QACtE,6EAA6E;QAC7E,4EAA4E;QAC5E,sEAAsE;QACtE,2EAA2E;QAC3E,+CAA+C;QAC/C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAC5C,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,gBAAgB,EACrB,KAAK,EACL,OAAO,EACP,IAAI,CAAC,SAAS,CACf,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,qBAAqB,CAAC,UAAU,CAAC,EAAE,CAAC;YAClD,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QAED,yEAAyE;QACzE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAC5C,OAAO,EACP,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,OAAO,EACP,IAAI,CAAC,SAAS,EACd,SAAS,EACT,IAAI,CAAC,MAAM,EACX,GAAG,CACJ,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/C,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QAED,8DAA8D;QAC9D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CACrC,OAAO,EACP,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,OAAO,EACP,IAAI,CAAC,SAAS,EACd,OAAO,EACP,IAAI,CAAC,OAAO,EACZ,SAAS,EACT,IAAI,CAAC,MAAM,EACX,GAAG,CACJ,CAAC;QACF,6CAA6C;QAC7C,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,mDAAmD;QACrD,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;YACxC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,QAAQ,CAAC,QAAgB,EAAE,KAAkB;QACzD,4EAA4E;QAC5E,0EAA0E;QAC1E,8EAA8E;QAC9E,0CAA0C;QAC1C,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;YAAE,OAAO;QAC5B,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC;YACH,0EAA0E;YAC1E,4DAA4D;YAC5D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAC1C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAClD,CAAC;YACF,IAAI,WAAW,EAAE,CAAC;gBAChB,sEAAsE;gBACtE,qEAAqE;gBACrE,0EAA0E;gBAC1E,mEAAmE;gBACnE,+DAA+D;gBAC/D,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC9B,OAAO;YACT,CAAC;YAED,yEAAyE;YACzE,0EAA0E;YAC1E,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAC5B,MAAM,GAAG,GAAuB;oBAC9B,EAAE;oBACF,KAAK;oBACL,QAAQ;oBACR,aAAa,EAAE,KAAK,IAAI,EAAE;wBACxB,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;4BACnB,MAAM,IAAI,cAAc,CAAC;gCACvB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG;gCACxB,aAAa,EAAE,CAAC,CAAC;gCACjB,OAAO,EAAE,IAAI;6BACd,CAAC,CAAC;wBACL,CAAC;wBACD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBACxC,CAAC;iBACF,CAAC;gBACF,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3B,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,EAC3D,EAAE,CACH,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,2EAA2E;YAC3E,uEAAuE;YACvE,2EAA2E;YAC3E,6EAA6E;YAC7E,0EAA0E;YAC1E,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC9B,IAAI,CAAC,IAAI,GAAG,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;YACvC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;gBACvC,IAAI,CAAC,KAAK,CAAC,SAAS,CAClB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CACxD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,uEAAuE;YACvE,0EAA0E;YAC1E,yEAAyE;YACzE,6DAA6D;YAC7D,IAAI,GAAG,YAAY,cAAc,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBACvB,OAAO;YACT,CAAC;YACD,wEAAwE;YACxE,2EAA2E;YAC3E,2EAA2E;YAC3E,2EAA2E;YAC3E,IAAI,GAAG,YAAY,0BAA0B,EAAE,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC,CAAC;gBACjE,OAAO;YACT,CAAC;YACD,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QACrD,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,UAAU,CAAC,QAAgB;QACvC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,gFAAgF;IACxE,KAAK,CAAC,iBAAiB,CAC7B,QAAgB,EAChB,KAAkB,EAClB,GAAY;QAEZ,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACtD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpC,IAAI,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,qEAAqE;YACrE,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;gBAC/B,YAAY,EAAE,IAAI,CAAC,MAAM;gBACzB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,OAAO,EAAE,KAAK,CAAC,QAAQ;gBACvB,OAAO,EAAE,KAAgC;gBACzC,YAAY,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;gBAC9D,UAAU,EAAE,OAAO;aACpB,CAAC,CAAC;YACH,0EAA0E;YAC1E,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACzD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,2EAA2E;YAC3E,6EAA6E;QAC/E,CAAC;IACH,CAAC;IAED,0DAA0D;IAC1D,KAAK,CAAC,OAAO;QACX,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAwB;QACjC,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,uBAAuB,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjB,IAAI,CAAC,IAAI;gBACT,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;aACnD,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,aAAa;QACf,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;YAC5D,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC3C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC;IACxC,CAAC;CACF"}

package/dist/worker/sweeper-worker.d.ts

@@ -0,0 +1,107 @@
+import type { DeadLetterSink } from "./dead-letter.js";
+import { ObservabilityHooks } from "./observability-hooks.js";
+import { type SingleActiveLeaseOpts } from "./single-active-lease.js";
+import type { SqlExecutor } from "./tx.js";
+import type { WorkerHandle, WorkerHwm, WorkerStopOptions, WorkerStopResult } from "./worker-handle.js";
+/**
+ * The context a sweeper job receives each tick. Carries the executor the job
+ * runs its delete + checkpoint advance on, the durable cursor it resumes from,
+ * and the fence guard it MUST call before any non-idempotent delete.
+ */
+export interface SweeperJobContext {
+    /** A pooled executor (the sweep is independent of any apply transaction; the
+     *  job MAY open its own tx via the caller-provided executor factory). */
+    exec: SqlExecutor;
+    /** The durable cursor the worker resumes from (null on cold start). */
+    lastStreamId: string | null;
+    /**
+     * THE side-effecting guard. The job MUST `await ctx.validateFence()` right
+     * before any non-idempotent delete: it re-reads the live lease and throws
+     * {@link FenceLostError} if the lease was lost/taken-over since the tick
+     * began — so a stale sweeper's late delete cannot land.
+     */
+    validateFence(): Promise<void>;
+}
+/** What a single sweep tick returns: how many rows it removed + the new cursor. */
+export interface SweeperJobResult {
+    /** Rows removed this tick (surfaced as `nodii.worker.swept_total`). */
+    swept: number;
+    /**
+     * The new durable cursor position to advance the checkpoint to (e.g. the
+     * max consumed_at swept, or a synthetic monotonic marker). Omit / null to
+     * leave the cursor unchanged (a no-op sweep).
+     */
+    lastStreamId?: string | null;
+}
+/** The injected unit of sweep work (e.g. `dedupe.sweep(...)`, a retention DELETE). */
+export interface SweeperJobDefinition {
+    /** Stable name for logs/metrics (the lease scope defaults to this). */
+    name: string;
+    /** Run exactly one sweep pass. MUST call `ctx.validateFence()` before delete. */
+    runOnce(ctx: SweeperJobContext): Promise<SweeperJobResult>;
+}
+export interface SweeperWorkerOpts {
+    /** The injected sweep job. */
+    job: SweeperJobDefinition;
+    /** Provides a fresh pooled executor per tick (e.g. `() => postgresJsExecutor(sql)`). */
+    executor: () => SqlExecutor | Promise<SqlExecutor>;
+    /** Lease config (service + scope + ttlMs). The renew loop runs inside it. */
+    lease: Omit<SingleActiveLeaseOpts, "onLeaseLost">;
+    /** The durable checkpoint store (resume cursor). */
+    checkpoint: import("./checkpoint-store.js").CheckpointStore;
+    /** The checkpoint subscription key (defaults to `sweeper:<job.name>`). */
+    subscription?: string;
+    /** Tick interval (ms) for the autonomous loop. Default 60_000. */
+    intervalMs?: number;
+    /** Injected dead-letter sink (NO default). A tick error is dead-lettered. */
+    deadLetter: DeadLetterSink;
+    /** Observability hooks (canonical metrics). Constructed if omitted. */
+    hooks?: ObservabilityHooks;
+    /** Worker label for metric attributes. Default "sweeper". */
+    workerLabel?: string;
+    /** Launch the interval loop on `start()`. Default true; tests pass false to
+     *  drive `runOnce()` deterministically. */
+    autoStart?: boolean;
+}
+/**
+ * SweeperWorker — the reference {@link WorkerHandle}. Acquires a single-active
+ * lease, runs the injected sweep job on an interval, advances the checkpoint
+ * each tick (resumable), and drains bounded on stop.
+ */
+export declare class SweeperWorker implements WorkerHandle {
+    private readonly job;
+    private readonly executorFactory;
+    private readonly lease;
+    private readonly checkpoint;
+    private readonly subscription;
+    private readonly intervalMs;
+    private readonly deadLetter;
+    private readonly hooks;
+    private readonly autoStart;
+    private _running;
+    private _stopped;
+    private _inFlight;
+    private _hwm;
+    private _lastAppliedAtMs;
+    private loopTimer;
+    constructor(opts: SweeperWorkerOpts);
+    get running(): boolean;
+    get leaseAcquired(): boolean;
+    get lag(): number | null;
+    get currentHwm(): WorkerHwm;
+    /** Acquire the lease + start the interval loop. Idempotent. */
+    start(): Promise<void>;
+    /** Read the durable checkpoint so a restart resumes rather than re-sweeps. */
+    private hydrateCursor;
+    /** One interval tick: acquire-if-needed → runOnce → swallow into dead-letter. */
+    private tick;
+    /**
+     * Run exactly ONE sweep pass. Acquires/validates the lease for the tick,
+     * invokes the injected job, advances the checkpoint, emits `swept_total`.
+     * Resolves the number of rows swept (0 when the lease is not held).
+     */
+    runOnce(): Promise<number>;
+    /** Bounded graceful stop: halt the loop, let an in-flight tick finish, release. */
+    stop(opts?: WorkerStopOptions): Promise<WorkerStopResult>;
+}
+//# sourceMappingURL=sweeper-worker.d.ts.map

package/dist/worker/sweeper-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sweeper-worker.d.ts","sourceRoot":"","sources":["../../src/worker/sweeper-worker.ts"],"names":[],"mappings":"AAuBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAIL,KAAK,qBAAqB,EAC3B,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EACV,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC;6EACyE;IACzE,IAAI,EAAE,WAAW,CAAC;IAClB,uEAAuE;IACvE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B;;;;;OAKG;IACH,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAChC;AAED,mFAAmF;AACnF,MAAM,WAAW,gBAAgB;IAC/B,uEAAuE;IACvE,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,sFAAsF;AACtF,MAAM,WAAW,oBAAoB;IACnC,uEAAuE;IACvE,IAAI,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,OAAO,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC5D;AAED,MAAM,WAAW,iBAAiB;IAChC,8BAA8B;IAC9B,GAAG,EAAE,oBAAoB,CAAC;IAC1B,wFAAwF;IACxF,QAAQ,EAAE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACnD,6EAA6E;IAC7E,KAAK,EAAE,IAAI,CAAC,qBAAqB,EAAE,aAAa,CAAC,CAAC;IAClD,oDAAoD;IACpD,UAAU,EAAE,OAAO,uBAAuB,EAAE,eAAe,CAAC;IAC5D,0EAA0E;IAC1E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kEAAkE;IAClE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,UAAU,EAAE,cAAc,CAAC;IAC3B,uEAAuE;IACvE,KAAK,CAAC,EAAE,kBAAkB,CAAC;IAC3B,6DAA6D;IAC7D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;+CAC2C;IAC3C,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAKD;;;;GAIG;AACH,qBAAa,aAAc,YAAW,YAAY;IAChD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAuB;IAC3C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA2C;IAC3E,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoB;IAC1C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAkD;IAC7E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IAEpC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,IAAI,CAAqC;IACjD,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,SAAS,CAA+C;gBAEpD,IAAI,EAAE,iBAAiB;IAyBnC,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,GAAG,IAAI,MAAM,GAAG,IAAI,CAGvB;IAED,IAAI,UAAU,IAAI,SAAS,CAE1B;IAED,+DAA+D;IACzD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAiB5B,8EAA8E;YAChE,aAAa;IAW3B,iFAAiF;YACnE,IAAI;IAwBlB;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC;IAyEhC,mFAAmF;IAC7E,IAAI,CAAC,IAAI,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAmBhE"}