@nodii/telemetry 0.13.0 → 0.14.1

package/dist/worker/streams-worker.js

@@ -0,0 +1,621 @@
+// D402 — StreamsWorker: the generic (non-replication) Redis-Streams consumer —
+// the canonical Pattern-2 begin/completion-event consumer.
+//
+// D402 (LOCKED): "StreamsWorker — generic (non-replication) stream consumer; the
+// canonical Pattern-2 begin/completion-event consumer." It closes the gap D402
+// names directly: "the Pattern-2 begin/completion-event consumer must be
+// StreamsWorker (today hand-rolled per service via
+// idempotency.wrapConsumerWorker)."
+//
+// ── Composition (which primitives, how) ──────────────────────────────────────
+//   - SingleActiveLease  — single-active per (service, stream); ttl/2 renew loop
+//     runs inside it. `validateFence(token)` guards the apply side-effect: a
+//     lease-lost worker mid-apply has its stale write REJECTED (the key test).
+//   - DedupeStore        — `seen(...)` skips an already-applied (event_id, topic)
+//     before any work; `record(...)` runs INSIDE the apply tx so the dedupe row
+//     commits together-or-none with the projection + the cursor advance.
+//   - CheckpointStore    — `advance(...)` runs in the SAME apply tx → HWM cursor
+//     never advances past an un-applied event.
+//   - ObservabilityHooks — lease/lag/hwm/depth metrics, canonical names.
+//   - SignalDrain        — `stop({timeoutMs})` is registerable; bounded drain.
+//   - DeadLetterSink     — INJECTED (no default); a poison event after retries
+//     is dead-lettered + XACK'd so the PEL doesn't grow unbounded.
+//
+// ── The consume sequence (per 01-comms § 9.2 / 09-replication § 4) ───────────
+//   XREADGROUP > → for each (id, event):
+//     [1] dedupe.seen? → XACK + skip (already applied; redelivery is a no-op)
+//     [2] apply tx: projection upsert + dedupe.record + checkpoint.advance, with
+//         validateFence(token) called BEFORE the side-effect (the fence guard)
+//     [3] XACK on success; on error retry-then-dead-letter.
+//
+// The apply itself is the INJECTED handler — the caller owns the projection
+// upsert (this lib doesn't know the consumer's schema). The handler receives the
+// tx executor + the dedupe/checkpoint primitives pre-bound so it runs everything
+// on ONE transaction.
+import { ensurePiiRedaction } from "./dead-letter.js";
+import { ObservabilityHooks } from "./observability-hooks.js";
+import { FenceCheckUnavailableError, FenceLostError, SingleActiveLease, } from "./single-active-lease.js";
+const DEFAULT_BATCH = 32;
+const DEFAULT_BLOCK_MS = 2_000;
+const DEFAULT_RETRY_MAX = 3;
+const DEFAULT_STOP_TIMEOUT_MS = 30_000;
+const DEFAULT_RECLAIM_MIN_IDLE_MS = 30_000;
+/**
+ * Wraps a throw that originated from the INJECTED apply handler (the consumer's
+ * projection upsert) so the worker can tell a POISON failure (the handler itself
+ * rejected on a well-formed event) apart from a TRANSIENT infra failure (the tx
+ * open / commit, the dedupe-store probe, or `dedupe.record` / `checkpoint.advance`
+ * threw because postgres is unreachable). ONLY a poison failure may travel the
+ * retry→dead-letter path; an untagged throw inside the apply path is infra and is
+ * left in the PEL to be retried once the store recovers (symmetric with
+ * {@link FenceCheckUnavailableError} on the redis-down fence path, and with the
+ * OutboxDispatcher, which survives a `begin()` throw without dead-lettering).
+ */
+export class PoisonApplyError extends Error {
+    /** The original error the apply handler threw. */
+    cause;
+    constructor(cause) {
+        super(cause instanceof Error ? cause.message : String(cause));
+        this.name = "PoisonApplyError";
+        this.cause = cause;
+    }
+}
+/** Unwrap a {@link PoisonApplyError} to the handler's original error. */
+function unwrapPoison(err) {
+    return err.cause;
+}
+/** Consumer-group name — `${service}:${stream}` (canonical per replica-consumer). */
+export function streamsConsumerGroup(service, stream) {
+    return `${service}:${stream}`;
+}
+/**
+ * Parse an ioredis XREADGROUP response into PARSED `(id, event)` pairs AND the
+ * structurally-unparseable entries (malformed JSON in the `event` field). An
+ * entry with NO `event` field is skipped (not malformed — nothing to apply); an
+ * entry WITH an `event` field that fails JSON.parse is MALFORMED (poison) so the
+ * worker can dead-letter + XACK it rather than orphan it in the PEL forever.
+ */
+export function parseStreamEntriesPartitioned(raw) {
+    const entries = [];
+    const malformed = [];
+    if (!Array.isArray(raw))
+        return { entries, malformed };
+    for (const streamEntry of raw) {
+        if (!Array.isArray(streamEntry) || streamEntry.length < 2)
+            continue;
+        const streamEntries = streamEntry[1];
+        if (!Array.isArray(streamEntries))
+            continue;
+        for (const e of streamEntries) {
+            if (!Array.isArray(e) || e.length < 2)
+                continue;
+            const id = String(e[0]);
+            const fields = e[1];
+            if (!Array.isArray(fields))
+                continue;
+            let payload;
+            for (let i = 0; i < fields.length - 1; i += 2) {
+                if (String(fields[i]) === "event") {
+                    payload = String(fields[i + 1]);
+                    break;
+                }
+            }
+            if (payload === undefined)
+                continue;
+            let parsed;
+            try {
+                parsed = JSON.parse(payload);
+            }
+            catch {
+                // Malformed `event` JSON — a POISON entry that can never be applied.
+                // Surface it so the worker dead-letters + XACKs it (never a silent drop /
+                // an orphan that re-cycles through XAUTOCLAIM on every reclaim).
+                malformed.push({ id, rawPayload: payload });
+                continue;
+            }
+            // Valid JSON but NOT an object (a bare array / number / string) can never
+            // be an event dict → also poison (parity with Go's `map[string]any`
+            // unmarshal + Python's dict check).
+            if (typeof parsed !== "object" ||
+                parsed === null ||
+                Array.isArray(parsed)) {
+                malformed.push({ id, rawPayload: payload });
+                continue;
+            }
+            entries.push({ id, event: parsed });
+        }
+    }
+    return { entries, malformed };
+}
+/** Parse an ioredis XREADGROUP response into flat (id, event) pairs (the
+ *  parsed entries only). Retained for API compatibility; the worker uses
+ *  {@link parseStreamEntriesPartitioned} so it can also dead-letter malformed
+ *  entries. */
+export function parseStreamEntries(raw) {
+    return parseStreamEntriesPartitioned(raw).entries;
+}
+/**
+ * Parse an ioredis XAUTOCLAIM reply `[cursor, entries, deleted]` into PARSED
+ * (id, event) pairs AND the malformed entries. `entries` has the same
+ * `[[id, [f, v, ...]], ...]` shape as one stream's entries in an XREADGROUP
+ * reply, so we wrap it and reuse {@link parseStreamEntriesPartitioned}.
+ */
+export function parseAutoClaimEntriesPartitioned(raw) {
+    if (!Array.isArray(raw) || raw.length < 2) {
+        return { entries: [], malformed: [] };
+    }
+    const claimed = raw[1];
+    if (!Array.isArray(claimed))
+        return { entries: [], malformed: [] };
+    // Wrap as a single-stream XREADGROUP shape: [[<stream>, <entries>]].
+    return parseStreamEntriesPartitioned([["", claimed]]);
+}
+/**
+ * Parse an ioredis XAUTOCLAIM reply `[cursor, entries, deleted]` into flat
+ * (id, event) pairs (parsed entries only). Retained for API compatibility.
+ */
+export function parseAutoClaimEntries(raw) {
+    return parseAutoClaimEntriesPartitioned(raw).entries;
+}
+/**
+ * StreamsWorker — the generic Pattern-2 single-active stream consumer. Composes
+ * lease + dedupe + checkpoint + hooks + drain to consume EXACTLY-ONCE.
+ */
+export class StreamsWorker {
+    redis;
+    stream;
+    apply;
+    begin;
+    dedupe;
+    checkpoint;
+    lease;
+    subscription;
+    deadLetter;
+    retryMaxAttempts;
+    pollBatch;
+    blockMs;
+    reclaimMinIdleMs;
+    hooks;
+    autoStart;
+    group;
+    consumer;
+    retries = new Map();
+    _running = false;
+    _stopped = false;
+    _inFlight = 0;
+    _hwm = { lastStreamId: null };
+    _lastAppliedAtMs = null;
+    groupEnsured = false;
+    loop = Promise.resolve();
+    constructor(opts) {
+        this.redis = opts.redis;
+        this.stream = opts.stream;
+        this.apply = opts.apply;
+        this.begin = opts.begin;
+        this.dedupe = opts.dedupe;
+        this.checkpoint = opts.checkpoint;
+        this.lease = new SingleActiveLease({
+            ...opts.lease,
+            onLeaseLost: () => this.hooks.leaseLost(),
+        });
+        this.subscription = opts.subscription ?? `streams:${opts.stream}`;
+        this.deadLetter = ensurePiiRedaction(opts.deadLetter);
+        this.retryMaxAttempts = opts.retryMaxAttempts ?? DEFAULT_RETRY_MAX;
+        this.pollBatch = opts.pollBatch ?? DEFAULT_BATCH;
+        this.blockMs = opts.blockMs ?? DEFAULT_BLOCK_MS;
+        this.reclaimMinIdleMs =
+            opts.reclaimMinIdleMs ??
+                Math.max(opts.lease.ttlMs, DEFAULT_RECLAIM_MIN_IDLE_MS);
+        this.autoStart = opts.autoStart !== false;
+        this.hooks =
+            opts.hooks ??
+                new ObservabilityHooks({
+                    attributes: {
+                        service: opts.lease.service,
+                        scope: opts.lease.scope,
+                        worker: opts.workerLabel ?? "streams",
+                    },
+                });
+        this.group = streamsConsumerGroup(opts.lease.service, opts.stream);
+        this.consumer = `${opts.lease.service}:${this.lease.ownerId}`;
+    }
+    get running() {
+        return this._running;
+    }
+    get leaseAcquired() {
+        return this.lease.leaseAcquired;
+    }
+    get lag() {
+        if (this._lastAppliedAtMs === null)
+            return null;
+        return Math.max(0, (Date.now() - this._lastAppliedAtMs) / 1000);
+    }
+    get currentHwm() {
+        return this._hwm;
+    }
+    /** Idempotently create the consumer group (swallow BUSYGROUP). */
+    async ensureGroup() {
+        if (this.groupEnsured)
+            return;
+        try {
+            await this.redis.xgroup("CREATE", this.stream, this.group, "$", "MKSTREAM");
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (!/BUSYGROUP/i.test(msg))
+                throw err;
+        }
+        this.groupEnsured = true;
+    }
+    async start() {
+        if (this._running)
+            return;
+        this._stopped = false;
+        const acquired = await this.lease.acquire();
+        if (acquired)
+            this.hooks.leaseAcquired();
+        // BUG-1 fix: if ANY boot step AFTER a successful acquire fails (ensureGroup
+        // throws — redis down at boot / a non-BUSYGROUP error), RELEASE the lease
+        // before rethrowing. A boot failure must NOT orphan the lease (a held key
+        // blocks a restart until the TTL lapses).
+        try {
+            await this.ensureGroup();
+            await this.hydrateCursor();
+        }
+        catch (err) {
+            if (acquired)
+                await this.lease.release();
+            throw err;
+        }
+        this._running = true;
+        // Background consume loop. `autoStart:false` lets a test drive `runOnce()`.
+        if (this.autoStart)
+            this.loop = this.runLoop();
+    }
+    async hydrateCursor() {
+        try {
+            await this.begin(async (tx) => {
+                const cp = await this.checkpoint.get(this.subscription, tx);
+                if (cp)
+                    this._hwm = { lastStreamId: cp.lastStreamId };
+            });
+        }
+        catch {
+            // Cold-start read failure — cursor stays null, consume starts from `>`.
+        }
+    }
+    async runLoop() {
+        while (!this._stopped) {
+            if (this.lease.lost)
+                break;
+            try {
+                const processed = await this.pollOnce();
+                if (processed === 0) {
+                    // Yield so a synchronous test-double doesn't tight-loop.
+                    await new Promise((r) => setTimeout(r, 0));
+                }
+            }
+            catch (err) {
+                // BUG-2 fix: a poll that throws BECAUSE we're shutting down (stop() set
+                // `_stopped` and may have torn down the redis client mid-poll) is a CLEAN
+                // cancel, NOT a redis failure — exit quietly (no error metric, no
+                // error-sleep). A genuine poll-level error (XREADGROUP rejected — Redis
+                // down) while still running is the Pillar-5 infra-failure case: log via
+                // metric + back off, NEVER crash the loop.
+                if (this._stopped)
+                    break;
+                this.hooks.recordDepth(-1, { error: "poll_failed" });
+                void err;
+                await new Promise((r) => setTimeout(r, 200));
+            }
+        }
+    }
+    /**
+     * Poll one batch + apply each entry. Returns count processed.
+     *
+     * Two reads per poll: FIRST our OWN pending-entries-list (PEL) at id `0` —
+     * entries delivered to this consumer but NOT yet XACK'd (a prior apply failed
+     * + NACK'd them); they MUST be retried, and a plain `>` read never re-returns
+     * a PEL entry. THEN `>` for brand-new entries. Without the PEL re-read a
+     * poison event would be applied exactly once and never retried.
+     */
+    async pollOnce() {
+        if (!this.lease.leaseAcquired) {
+            const got = await this.lease.acquire();
+            if (got) {
+                this.hooks.leaseAcquired();
+                // Re-ensure the consumer group on every re-acquire: a Redis failover
+                // across the prior holder's death may have wiped the group, so a cached
+                // `groupEnsured` would let xreadgroup hit NOGROUP forever. Re-running is a
+                // harmless BUSYGROUP no-op when the group still exists.
+                this.groupEnsured = false;
+                // BUG fix (in-loop re-acquire): if ensureGroup throws AFTER this poll
+                // re-acquired the lease, RELEASE it before rethrowing — otherwise the
+                // renewal loop holds + renews the lease INDEFINITELY while the next poll
+                // skips this block (leaseAcquired is true) and hammers a never-created
+                // group (NOGROUP loop), deadlocking every other instance.
+                try {
+                    await this.ensureGroup();
+                }
+                catch (err) {
+                    await this.lease.release();
+                    throw err;
+                }
+            }
+            else {
+                return 0;
+            }
+        }
+        let processed = 0;
+        // [reclaim] CRASH-RECOVERY reaper: XAUTOCLAIM transfers PENDING entries idle
+        // ≥ reclaimMinIdleMs from a DEAD predecessor (a crashed worker with a
+        // different consumer name) to US, the single-active successor. Without this,
+        // a crashed worker's un-ACK'd in-flight entries are PERMANENTLY ORPHANED (a
+        // plain "0" PEL re-read is per-consumer) → silent event loss → broken
+        // at-least-once. The fence guard inside the apply tx still rejects a stale
+        // write if a still-alive predecessor races us.
+        const claimedRaw = await this.redis.xautoclaim(this.stream, this.group, this.consumer, this.reclaimMinIdleMs, "0-0", "COUNT", this.pollBatch);
+        const claimed = parseAutoClaimEntriesPartitioned(claimedRaw);
+        for (const p of claimed.entries) {
+            if (this._stopped || this.lease.lost)
+                break;
+            await this.applyOne(p.id, p.event);
+            processed += 1;
+        }
+        processed += await this.handleMalformedEntries(claimed.malformed);
+        // [retry] Re-read our own PEL (id `0`, non-blocking) for NACK'd entries.
+        const pendingRaw = await this.redis.xreadgroup("GROUP", this.group, this.consumer, "COUNT", this.pollBatch, "STREAMS", this.stream, "0");
+        const pending = parseStreamEntriesPartitioned(pendingRaw);
+        for (const p of pending.entries) {
+            if (this._stopped || this.lease.lost)
+                break;
+            await this.applyOne(p.id, p.event);
+            processed += 1;
+        }
+        processed += await this.handleMalformedEntries(pending.malformed);
+        // [new] Read brand-new entries (`>`, blocking up to blockMs).
+        const raw = await this.redis.xreadgroup("GROUP", this.group, this.consumer, "COUNT", this.pollBatch, "BLOCK", this.blockMs, "STREAMS", this.stream, ">");
+        // Sample backlog depth (cheap; best-effort).
+        try {
+            const depth = await this.redis.xlen(this.stream);
+            this.hooks.recordDepth(depth);
+        }
+        catch {
+            /* depth is advisory — never fail the poll on it */
+        }
+        const fresh = parseStreamEntriesPartitioned(raw);
+        for (const p of fresh.entries) {
+            if (this._stopped || this.lease.lost)
+                break;
+            await this.applyOne(p.id, p.event);
+            processed += 1;
+        }
+        processed += await this.handleMalformedEntries(fresh.malformed);
+        return processed;
+    }
+    /**
+     * Dead-letter + XACK every structurally-unparseable (malformed-JSON) entry. A
+     * malformed entry is a POISON message — it can NEVER be applied (there is no
+     * decodable event) — so leaving it in the PEL only re-cycles it through
+     * XAUTOCLAIM on every reclaim, forever. We route the raw bytes through the
+     * (PII-redacting) dead-letter sink with a parse-error message, then XACK it so
+     * it leaves the PEL. If the sink itself fails, we leave the entry in the PEL
+     * (recoverable; never silently dropped) — exactly like a failed poison DLQ.
+     * Returns the count handled (so the poll's processed count reflects them).
+     */
+    async handleMalformedEntries(malformed) {
+        let handled = 0;
+        for (const m of malformed) {
+            if (this._stopped || this.lease.lost)
+                break;
+            try {
+                await this.deadLetter.deadLetter({
+                    sourceStream: this.stream,
+                    topic: "",
+                    eventId: "",
+                    payload: { raw: m.rawPayload },
+                    errorMessage: "malformed stream entry: event field is not valid JSON",
+                    retryCount: 0,
+                });
+                // XACK so the poison entry leaves the PEL (no orphan / no re-cycle).
+                await this.redis.xack(this.stream, this.group, m.id);
+                this.retries.delete(m.id);
+                handled += 1;
+            }
+            catch {
+                // The dead-letter sink failed — leave the entry in the PEL (recoverable)
+                // rather than XACK + lose it. Never XACK on a failed dead-letter.
+                this.hooks.recordDepth(-1, { error: "malformed_dead_letter_failed" });
+            }
+        }
+        return handled;
+    }
+    /**
+     * Apply one entry: dedupe-skip → apply-tx (projection + dedupe.record +
+     * checkpoint.advance, fence-guarded) → XACK. On error, retry-then-dead-letter.
+     */
+    async applyOne(streamId, event) {
+        // We dropped the lease mid-flight; the entry stays un-ACK'd in our PEL. The
+        // single-active successor (a new instance with a DIFFERENT consumer name)
+        // reclaims it via the [reclaim] XAUTOCLAIM reaper once idle — NOT via a plain
+        // "0" PEL re-read (that is per-consumer).
+        if (this.lease.lost)
+            return;
+        this._inFlight += 1;
+        const token = this.lease.fencingToken;
+        try {
+            // [1] Pre-tx dedupe probe (cheap). A redelivered already-applied event is
+            //     XACK'd + skipped (exactly-once on the consumer side).
+            const alreadySeen = await this.begin((tx) => this.dedupe.seen(event.event_id, event.topic, tx));
+            if (alreadySeen) {
+                // The event already committed (durable dedupe row) — only the XACK is
+                // outstanding. A failing XACK here is a TRANSIENT ack-failure, NOT a
+                // poison event: swallow it (the entry stays un-acked in the PEL, re-acked
+                // next poll once redis recovers). NEVER route a committed event to
+                // retry/DLQ over an ack blip (matches Go's `_, _ = Ack(...)`).
+                await this.ackSwallow(streamId);
+                this.retries.delete(streamId);
+                return;
+            }
+            // [2] Apply tx — projection upsert + dedupe.record + checkpoint.advance,
+            //     all on ONE tx (atomic). The fence guard runs BEFORE the projection.
+            await this.begin(async (tx) => {
+                const ctx = {
+                    tx,
+                    event,
+                    streamId,
+                    validateFence: async () => {
+                        if (token === null) {
+                            throw new FenceLostError({
+                                leaseKey: this.lease.key,
+                                expectedToken: -1,
+                                current: null,
+                            });
+                        }
+                        await this.lease.validateFence(token);
+                    },
+                };
+                // TAG the apply handler's throws as POISON so the catch can distinguish a
+                // bad-event handler rejection (→ retry/DLQ) from a TRANSIENT store-down
+                // throw of the surrounding tx machinery / dedupe-record / cursor-advance
+                // (→ leave in PEL, retry on recovery). A FenceLostError raised inside the
+                // handler's validateFence is re-thrown bare (NOT wrapped) so the outer
+                // FenceLostError branch still sees it.
+                try {
+                    await this.apply(ctx);
+                }
+                catch (handlerErr) {
+                    if (handlerErr instanceof FenceLostError ||
+                        handlerErr instanceof FenceCheckUnavailableError) {
+                        throw handlerErr;
+                    }
+                    throw new PoisonApplyError(handlerErr);
+                }
+                await this.dedupe.record(event.event_id, event.topic, tx);
+                await this.checkpoint.advance({ subscription: this.subscription, lastStreamId: streamId }, tx);
+            });
+            // [3] Commit succeeded → XACK + advance local HWM. The XACK is POST-COMMIT
+            //     and NOT part of the apply tx: a failing XACK here is a transient
+            //     ack-failure, NOT a poison event — swallow it (the committed event is
+            //     re-acked next poll; the dedupe row makes a redelivery a no-op). Do NOT
+            //     route a successfully-committed event to retry/DLQ over an ack blip.
+            await this.ackSwallow(streamId);
+            this.retries.delete(streamId);
+            this._hwm = { lastStreamId: streamId };
+            this._lastAppliedAtMs = Date.now();
+            if (event.occurred_at_ms !== undefined) {
+                this.hooks.recordLag(Math.max(0, (Date.now() - event.occurred_at_ms) / 1000));
+            }
+            else {
+                this.hooks.recordLag(0);
+            }
+        }
+        catch (err) {
+            // A FenceLostError means the lease was lost/taken-over mid-apply — the
+            // stale write was REJECTED before it landed. Do NOT XACK (the entry stays
+            // in our PEL; the single-active successor reclaims it via the XAUTOCLAIM
+            // reaper once idle) and do NOT dead-letter (benign handoff).
+            if (err instanceof FenceLostError) {
+                this.hooks.leaseLost();
+                return;
+            }
+            // A FenceCheckUnavailableError means the fence-check READ failed (redis
+            // down) — TRANSIENT. Fail closed (the write was withheld), but do NOT XACK
+            // and do NOT dead-letter a non-poison event over a redis blip: leave it in
+            // the PEL to be reprocessed once redis recovers (cross-runtime-identical).
+            if (err instanceof FenceCheckUnavailableError) {
+                this.hooks.recordDepth(-1, { error: "fence_check_unavailable" });
+                return;
+            }
+            // A POISON failure — the injected apply handler itself rejected on a
+            // well-formed event — is the ONLY thing that travels the retry→DLQ path.
+            if (err instanceof PoisonApplyError) {
+                await this.retryOrDeadLetter(streamId, event, unwrapPoison(err));
+                return;
+            }
+            // ANY OTHER throw inside the apply path is a TRANSIENT INFRA failure: the
+            // pre-tx dedupe-store probe, the tx open/commit, `dedupe.record`, or
+            // `checkpoint.advance` threw because postgres/the store is unreachable. A
+            // healthy event must NOT be dead-lettered over a store outage — treat it
+            // exactly like the redis-down fence path: do NOT bump the retry ledger, do
+            // NOT XACK, do NOT dead-letter; leave it in the PEL and retry on the next
+            // poll once the store recovers (symmetric with the OutboxDispatcher, which
+            // survives a `begin()` throw without dead-lettering).
+            this.hooks.recordDepth(-1, { error: "transient_infra" });
+            return;
+        }
+        finally {
+            this._inFlight -= 1;
+        }
+    }
+    /**
+     * XACK an already-committed entry, SWALLOWING a transient ack failure. The
+     * XACK is post-commit (or post-dedupe-skip) and NOT part of the apply tx — a
+     * failing XACK is a transient redis blip, never a poison event. On failure the
+     * entry simply stays un-acked in the PEL and is re-acked next poll once redis
+     * recovers (the durable dedupe row makes the redelivery a no-op). This is what
+     * Go does (`_, _ = Ack(...)`); a throw here must NEVER reach retry/DLQ.
+     */
+    async ackSwallow(streamId) {
+        try {
+            await this.redis.xack(this.stream, this.group, streamId);
+        }
+        catch {
+            this.hooks.recordDepth(-1, { error: "ack_failed" });
+        }
+    }
+    /** Increment the retry ledger; dead-letter (+XACK) once the budget is spent. */
+    async retryOrDeadLetter(streamId, event, err) {
+        const attempt = (this.retries.get(streamId) ?? 0) + 1;
+        this.retries.set(streamId, attempt);
+        if (attempt < this.retryMaxAttempts) {
+            // No XACK — the entry stays in the PEL and is redelivered next poll.
+            return;
+        }
+        try {
+            await this.deadLetter.deadLetter({
+                sourceStream: this.stream,
+                topic: event.topic,
+                eventId: event.event_id,
+                payload: event,
+                errorMessage: err instanceof Error ? err.message : String(err),
+                retryCount: attempt,
+            });
+            // ACK after the dead-letter is durable so the PEL doesn't grow unbounded.
+            await this.redis.xack(this.stream, this.group, streamId);
+            this.retries.delete(streamId);
+        }
+        catch {
+            // The dead-letter sink itself failed — leave the entry in the PEL (it will
+            // be redelivered) rather than losing it. Never XACK on a failed dead-letter.
+        }
+    }
+    /** Run exactly one poll batch (test seam / cron tick). */
+    async runOnce() {
+        return this.pollOnce();
+    }
+    async stop(opts) {
+        const timeoutMs = opts?.timeoutMs ?? DEFAULT_STOP_TIMEOUT_MS;
+        const start = Date.now();
+        this._stopped = true;
+        this._running = false;
+        try {
+            await Promise.race([
+                this.loop,
+                new Promise((r) => setTimeout(r, timeoutMs)),
+            ]);
+        }
+        catch {
+            /* swallow */
+        }
+        while (this._inFlight > 0 && Date.now() - start < timeoutMs) {
+            await new Promise((r) => setTimeout(r, 10));
+        }
+        const drainedCleanly = this._inFlight === 0;
+        await this.lease.release();
+        const durationMs = Date.now() - start;
+        this.hooks.recordDrainDuration(durationMs);
+        return { drainedCleanly, durationMs };
+    }
+}
+//# sourceMappingURL=streams-worker.js.map

package/dist/worker/streams-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"streams-worker.js","sourceRoot":"","sources":["../../src/worker/streams-worker.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,2DAA2D;AAC3D,EAAE;AACF,iFAAiF;AACjF,+EAA+E;AAC/E,yEAAyE;AACzE,mDAAmD;AACnD,oCAAoC;AACpC,EAAE;AACF,gFAAgF;AAChF,iFAAiF;AACjF,6EAA6E;AAC7E,+EAA+E;AAC/E,kFAAkF;AAClF,gFAAgF;AAChF,yEAAyE;AACzE,iFAAiF;AACjF,+CAA+C;AAC/C,yEAAyE;AACzE,+EAA+E;AAC/E,+EAA+E;AAC/E,mEAAmE;AACnE,EAAE;AACF,gFAAgF;AAChF,yCAAyC;AACzC,8EAA8E;AAC9E,iFAAiF;AACjF,+EAA+E;AAC/E,4DAA4D;AAC5D,EAAE;AACF,4EAA4E;AAC5E,iFAAiF;AACjF,iFAAiF;AACjF,sBAAsB;AAGtB,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAGtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,0BAA0B,EAC1B,cAAc,EACd,iBAAiB,GAElB,MAAM,0BAA0B,CAAC;AA+GlC,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAC/B,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,uBAAuB,GAAG,MAAM,CAAC;AACvC,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAE3C;;;;;;;;;;GAUG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,kDAAkD;IACzC,KAAK,CAAU;IACxB,YAAY,KAAc;QACxB,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF;AAED,yEAAyE;AACzE,SAAS,YAAY,CAAC,GAAqB;IACzC,OAAO,GAAG,CAAC,KAAK,CAAC;AACnB,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,oBAAoB,CAAC,OAAe,EAAE,MAAc;IAClE,OAAO,GAAG,OAAO,IAAI,MAAM,EAAE,CAAC;AAChC,CAAC;AAYD;;;;;;GAMG;AACH,MAAM,UAAU,6BAA6B,CAAC,GAAY;IAIxD,MAAM,OAAO,GAA8C,EAAE,CAAC;IAC9D,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IACvD,KAAK,MAAM,WAAW,IAAI,GAAgB,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QACpE,MAAM,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;YAAE,SAAS;QAC5C,KAAK,MAAM,CAAC,IAAI,aAA0B,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAChD,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,SAAS;YACrC,IAAI,OAA2B,CAAC;YAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;oBAClC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,OAAO,KAAK,SAAS;gBAAE,SAAS;YACpC,IAAI,MAAe,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC;gBACP,qEAAqE;gBACrE,0EAA0E;gBAC1E,iEAAiE;gBACjE,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC5C,SAAS;YACX,CAAC;YACD,0EAA0E;YAC1E,oEAAoE;YACpE,oCAAoC;YACpC,IACE,OAAO,MAAM,KAAK,QAAQ;gBAC1B,MAAM,KAAK,IAAI;gBACf,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EACrB,CAAC;gBACD,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC5C,SAAS;YACX,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAqB,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED;;;eAGe;AACf,MAAM,UAAU,kBAAkB,CAChC,GAAY;IAEZ,OAAO,6BAA6B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gCAAgC,CAAC,GAAY;IAI3D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1C,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACxC,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACvB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACnE,qEAAqE;IACrE,OAAO,6BAA6B,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,GAAY;IAEZ,OAAO,gCAAgC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;AACvD,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,aAAa;IACP,KAAK,CAAmB;IACxB,MAAM,CAAS;IACf,KAAK,CAAqB;IAC1B,KAAK,CAEN;IACC,MAAM,CAAc;IACpB,UAAU,CAAkB;IAC5B,KAAK,CAAoB;IACzB,YAAY,CAAS;IACrB,UAAU,CAAiB;IAC3B,gBAAgB,CAAS;IACzB,SAAS,CAAS;IAClB,OAAO,CAAS;IAChB,gBAAgB,CAAS;IACzB,KAAK,CAAqB;IAC1B,SAAS,CAAU;IACnB,KAAK,CAAS;IACd,QAAQ,CAAS;IAEjB,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,QAAQ,GAAG,KAAK,CAAC;IACjB,QAAQ,GAAG,KAAK,CAAC;IACjB,SAAS,GAAG,CAAC,CAAC;IACd,IAAI,GAAc,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACzC,gBAAgB,GAAkB,IAAI,CAAC;IACvC,YAAY,GAAG,KAAK,CAAC;IACrB,IAAI,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEhD,YAAY,IAAuB;QACjC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,IAAI,iBAAiB,CAAC;YACjC,GAAG,IAAI,CAAC,KAAK;YACb,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;SAC1C,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE,CAAC;QAClE,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,aAAa,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,gBAAgB,CAAC;QAChD,IAAI,CAAC,gBAAgB;YACnB,IAAI,CAAC,gBAAgB;gBACrB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC;QAC1D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC;QAC1C,IAAI,CAAC,KAAK;YACR,IAAI,CAAC,KAAK;gBACV,IAAI,kBAAkB,CAAC;oBACrB,UAAU,EAAE;wBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;wBAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;wBACvB,MAAM,EAAE,IAAI,CAAC,WAAW,IAAI,SAAS;qBACtC;iBACF,CAAC,CAAC;QACL,IAAI,CAAC,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACnE,IAAI,CAAC,QAAQ,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;IAChE,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;IAClC,CAAC;IAED,IAAI,GAAG;QACL,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,kEAAkE;IAC1D,KAAK,CAAC,WAAW;QACvB,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QAC9B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CACrB,QAAQ,EACR,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,GAAG,EACH,UAAU,CACX,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,MAAM,GAAG,CAAC;QACzC,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC5C,IAAI,QAAQ;YAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;QACzC,4EAA4E;QAC5E,0EAA0E;QAC1E,0EAA0E;QAC1E,0CAA0C;QAC1C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,QAAQ;gBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACzC,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,4EAA4E;QAC5E,IAAI,IAAI,CAAC,SAAS;YAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAC5B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAC5D,IAAI,EAAE;oBAAE,IAAI,CAAC,IAAI,GAAG,EAAE,YAAY,EAAE,EAAE,CAAC,YAAY,EAAE,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,wEAAwE;QAC1E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC3B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACxC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;oBACpB,yDAAyD;oBACzD,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,wEAAwE;gBACxE,0EAA0E;gBAC1E,kEAAkE;gBAClE,wEAAwE;gBACxE,wEAAwE;gBACxE,2CAA2C;gBAC3C,IAAI,IAAI,CAAC,QAAQ;oBAAE,MAAM;gBACzB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;gBACrD,KAAK,GAAG,CAAC;gBACT,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,QAAQ;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACvC,IAAI,GAAG,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;gBAC3B,qEAAqE;gBACrE,wEAAwE;gBACxE,2EAA2E;gBAC3E,wDAAwD;gBACxD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;gBAC1B,sEAAsE;gBACtE,sEAAsE;gBACtE,yEAAyE;gBACzE,uEAAuE;gBACvE,0DAA0D;gBAC1D,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC3B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;oBAC3B,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,CAAC;YACX,CAAC;QACH,CAAC;QACD,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,6EAA6E;QAC7E,sEAAsE;QACtE,6EAA6E;QAC7E,4EAA4E;QAC5E,sEAAsE;QACtE,2EAA2E;QAC3E,+CAA+C;QAC/C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAC5C,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,gBAAgB,EACrB,KAAK,EACL,OAAO,EACP,IAAI,CAAC,SAAS,CACf,CAAC;QACF,MAAM,OAAO,GAAG,gCAAgC,CAAC,UAAU,CAAC,CAAC;QAC7D,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,SAAS,IAAI,MAAM,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAElE,yEAAyE;QACzE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAC5C,OAAO,EACP,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,OAAO,EACP,IAAI,CAAC,SAAS,EACd,SAAS,EACT,IAAI,CAAC,MAAM,EACX,GAAG,CACJ,CAAC;QACF,MAAM,OAAO,GAAG,6BAA6B,CAAC,UAAU,CAAC,CAAC;QAC1D,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,SAAS,IAAI,MAAM,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAElE,8DAA8D;QAC9D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CACrC,OAAO,EACP,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,OAAO,EACP,IAAI,CAAC,SAAS,EACd,OAAO,EACP,IAAI,CAAC,OAAO,EACZ,SAAS,EACT,IAAI,CAAC,MAAM,EACX,GAAG,CACJ,CAAC;QACF,6CAA6C;QAC7C,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,mDAAmD;QACrD,CAAC;QACD,MAAM,KAAK,GAAG,6BAA6B,CAAC,GAAG,CAAC,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,SAAS,IAAI,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;;;OASG;IACK,KAAK,CAAC,sBAAsB,CAClC,SAAiC;QAEjC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;oBAC/B,YAAY,EAAE,IAAI,CAAC,MAAM;oBACzB,KAAK,EAAE,EAAE;oBACT,OAAO,EAAE,EAAE;oBACX,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,UAAU,EAAE;oBAC9B,YAAY,EAAE,uDAAuD;oBACrE,UAAU,EAAE,CAAC;iBACd,CAAC,CAAC;gBACH,qEAAqE;gBACrE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC1B,OAAO,IAAI,CAAC,CAAC;YACf,CAAC;YAAC,MAAM,CAAC;gBACP,yEAAyE;gBACzE,kEAAkE;gBAClE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,8BAA8B,EAAE,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,QAAQ,CAAC,QAAgB,EAAE,KAAkB;QACzD,4EAA4E;QAC5E,0EAA0E;QAC1E,8EAA8E;QAC9E,0CAA0C;QAC1C,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;YAAE,OAAO;QAC5B,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC;YACH,0EAA0E;YAC1E,4DAA4D;YAC5D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAC1C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAClD,CAAC;YACF,IAAI,WAAW,EAAE,CAAC;gBAChB,sEAAsE;gBACtE,qEAAqE;gBACrE,0EAA0E;gBAC1E,mEAAmE;gBACnE,+DAA+D;gBAC/D,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC9B,OAAO;YACT,CAAC;YAED,yEAAyE;YACzE,0EAA0E;YAC1E,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAC5B,MAAM,GAAG,GAAuB;oBAC9B,EAAE;oBACF,KAAK;oBACL,QAAQ;oBACR,aAAa,EAAE,KAAK,IAAI,EAAE;wBACxB,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;4BACnB,MAAM,IAAI,cAAc,CAAC;gCACvB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG;gCACxB,aAAa,EAAE,CAAC,CAAC;gCACjB,OAAO,EAAE,IAAI;6BACd,CAAC,CAAC;wBACL,CAAC;wBACD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBACxC,CAAC;iBACF,CAAC;gBACF,0EAA0E;gBAC1E,wEAAwE;gBACxE,yEAAyE;gBACzE,0EAA0E;gBAC1E,uEAAuE;gBACvE,uCAAuC;gBACvC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,IACE,UAAU,YAAY,cAAc;wBACpC,UAAU,YAAY,0BAA0B,EAChD,CAAC;wBACD,MAAM,UAAU,CAAC;oBACnB,CAAC;oBACD,MAAM,IAAI,gBAAgB,CAAC,UAAU,CAAC,CAAC;gBACzC,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3B,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,EAC3D,EAAE,CACH,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,2EAA2E;YAC3E,uEAAuE;YACvE,2EAA2E;YAC3E,6EAA6E;YAC7E,0EAA0E;YAC1E,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC9B,IAAI,CAAC,IAAI,GAAG,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;YACvC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;gBACvC,IAAI,CAAC,KAAK,CAAC,SAAS,CAClB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CACxD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,uEAAuE;YACvE,0EAA0E;YAC1E,yEAAyE;YACzE,6DAA6D;YAC7D,IAAI,GAAG,YAAY,cAAc,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBACvB,OAAO;YACT,CAAC;YACD,wEAAwE;YACxE,2EAA2E;YAC3E,2EAA2E;YAC3E,2EAA2E;YAC3E,IAAI,GAAG,YAAY,0BAA0B,EAAE,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC,CAAC;gBACjE,OAAO;YACT,CAAC;YACD,qEAAqE;YACrE,yEAAyE;YACzE,IAAI,GAAG,YAAY,gBAAgB,EAAE,CAAC;gBACpC,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;gBACjE,OAAO;YACT,CAAC;YACD,0EAA0E;YAC1E,qEAAqE;YACrE,0EAA0E;YAC1E,yEAAyE;YACzE,2EAA2E;YAC3E,0EAA0E;YAC1E,2EAA2E;YAC3E,sDAAsD;YACtD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACzD,OAAO;QACT,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,UAAU,CAAC,QAAgB;QACvC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,gFAAgF;IACxE,KAAK,CAAC,iBAAiB,CAC7B,QAAgB,EAChB,KAAkB,EAClB,GAAY;QAEZ,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACtD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpC,IAAI,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,qEAAqE;YACrE,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;gBAC/B,YAAY,EAAE,IAAI,CAAC,MAAM;gBACzB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,OAAO,EAAE,KAAK,CAAC,QAAQ;gBACvB,OAAO,EAAE,KAAgC;gBACzC,YAAY,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;gBAC9D,UAAU,EAAE,OAAO;aACpB,CAAC,CAAC;YACH,0EAA0E;YAC1E,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACzD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,2EAA2E;YAC3E,6EAA6E;QAC/E,CAAC;IACH,CAAC;IAED,0DAA0D;IAC1D,KAAK,CAAC,OAAO;QACX,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAwB;QACjC,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,uBAAuB,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjB,IAAI,CAAC,IAAI;gBACT,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;aACnD,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,aAAa;QACf,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;YAC5D,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC3C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC;IACxC,CAAC;CACF"}