@nodii/telemetry 0.12.0 → 0.14.0

package/dist/worker/observability-hooks.js

@@ -0,0 +1,189 @@
+// D402 — ObservabilityHooks: the canonical worker-metric vocabulary, routed
+// through telemetry's REAL `recordMetric` API (NOT a NOOP sink by default).
+//
+// D402 (LOCKED) — "ObservabilityHooks — lag / HWM / depth metrics with canonical
+// names." The orchestrator's task pins the exact canonical names:
+//   - nodii.worker.lease.acquired_total   (counter)
+//   - nodii.worker.lease.renewed_total    (counter)
+//   - nodii.worker.lease.lost_total       (counter)
+//   - nodii.worker.lag                     (gauge,  seconds)
+//   - nodii.worker.current_hwm             (gauge)
+//   - nodii.worker.depth                   (gauge — backlog/stream depth)
+//   - nodii.worker.drain.duration_ms       (histogram, ms)
+//   - nodii.worker.swept_total             (counter — sweepers only)
+//
+// These names live under the existing D18/D197 `async_worker` metric family
+// (background workers are the async_worker family). A consuming service merges
+// `WORKER_SUBSTRATE_VOCABULARY` (below) into `initTelemetry({ vocabularyRegistry
+// })` exactly as it merges `REPLICA_CONSUMER_VOCABULARY` — otherwise the
+// canonical names trip `MetricVocabularyError` (vocabulary is HARD per D24).
+//
+// ── Why this routes through `recordMetric`, never a default no-op ────────────
+// R1 (never-defer) forbids a Noop sink wired as the default. The hooks bind to
+// telemetry's own `recordMetric({ kind, name, value, attributes })`. That fn
+// is the real metric path: it validates the name against the loaded vocabulary
+// + the family + the kind, then records onto the live OTLP-backed metrics
+// backend. The ONLY injection seam is for tests, which `_setMetricsBackend(...)`
+// a capturing backend underneath — the hook itself always calls the real API.
+import { recordMetric } from "../metrics.js";
+/** Canonical worker-substrate metric names, pinned by D402 + the task. */
+export const WORKER_METRIC_NAMES = {
+    leaseAcquired: "nodii.worker.lease.acquired_total",
+    leaseRenewed: "nodii.worker.lease.renewed_total",
+    leaseLost: "nodii.worker.lease.lost_total",
+    lag: "nodii.worker.lag",
+    currentHwm: "nodii.worker.current_hwm",
+    depth: "nodii.worker.depth",
+    drainDurationMs: "nodii.worker.drain.duration_ms",
+    swept: "nodii.worker.swept_total",
+};
+/**
+ * The locked worker-substrate metric vocabulary — merge into
+ * `initTelemetry({ vocabularyRegistry: { ...svc, ...WORKER_SUBSTRATE_VOCABULARY } })`
+ * so the canonical names register under family `async_worker` instead of
+ * tripping `MetricVocabularyError`. Mirrors how `REPLICA_CONSUMER_VOCABULARY`
+ * is consumed.
+ */
+export const WORKER_SUBSTRATE_VOCABULARY = {
+    [WORKER_METRIC_NAMES.leaseAcquired]: {
+        family: "async_worker",
+        kind: "counter",
+        description: "Single-active lease acquisitions (a worker took the lease for its scope).",
+        decision: "D402",
+    },
+    [WORKER_METRIC_NAMES.leaseRenewed]: {
+        family: "async_worker",
+        kind: "counter",
+        description: "Single-active lease renewals at ttl/2 (CAS held).",
+        decision: "D402",
+    },
+    [WORKER_METRIC_NAMES.leaseLost]: {
+        family: "async_worker",
+        kind: "counter",
+        description: "Single-active lease losses (renew CAS failed / lease taken over / errored).",
+        decision: "D402",
+    },
+    [WORKER_METRIC_NAMES.lag]: {
+        family: "async_worker",
+        kind: "gauge",
+        unit: "seconds",
+        description: "Wall-clock lag between source emission and consumer apply for the worker.",
+        decision: "D402",
+    },
+    [WORKER_METRIC_NAMES.currentHwm]: {
+        family: "async_worker",
+        kind: "gauge",
+        description: "Current durable high-water-mark (checkpoint cursor position) the worker has advanced to.",
+        decision: "D402",
+    },
+    [WORKER_METRIC_NAMES.depth]: {
+        family: "async_worker",
+        kind: "gauge",
+        description: "Backlog / stream depth pending for the worker (unprocessed entries).",
+        decision: "D402",
+    },
+    [WORKER_METRIC_NAMES.drainDurationMs]: {
+        family: "async_worker",
+        kind: "histogram",
+        unit: "ms",
+        description: "Graceful-drain wall-clock duration on stop() (flush in-flight + release leases).",
+        decision: "D402",
+    },
+    [WORKER_METRIC_NAMES.swept]: {
+        family: "async_worker",
+        kind: "counter",
+        description: "Rows removed by a SweeperWorker retention sweep.",
+        decision: "D402",
+    },
+};
+/** Strip `undefined`-valued attributes so the recordMetric label map is clean. */
+function cleanAttributes(attrs, extra) {
+    const out = {};
+    for (const [k, v] of Object.entries({ ...attrs, ...extra })) {
+        if (v !== undefined)
+            out[k] = v;
+    }
+    return out;
+}
+/**
+ * ObservabilityHooks emits the canonical worker-substrate metrics through the
+ * real telemetry metric API. One instance per worker; the worker types call
+ * the typed methods at the corresponding lifecycle points (lease acquire/renew/
+ * lose, per-tick lag/hwm/depth refresh, drain on stop, sweep count).
+ *
+ * Usage:
+ *   const hooks = new ObservabilityHooks({
+ *     attributes: { service: "billing", scope: "outbox", worker: "outbox" },
+ *   });
+ *   hooks.leaseAcquired();
+ *   hooks.recordLag(12.5);
+ *   hooks.recordDrainDuration(842);
+ */
+export class ObservabilityHooks {
+    attrs;
+    record;
+    constructor(opts) {
+        this.attrs = opts.attributes;
+        // Default = the REAL recordMetric. R1: never a no-op default.
+        this.record = opts.record ?? recordMetric;
+    }
+    emit(kind, name, value, extra) {
+        this.record({
+            kind,
+            name,
+            value,
+            attributes: cleanAttributes(this.attrs, extra),
+        });
+    }
+    /** The worker acquired its single-active lease. */
+    leaseAcquired(extra) {
+        this.emit("counter", WORKER_METRIC_NAMES.leaseAcquired, 1, extra);
+    }
+    /** The worker renewed its lease (CAS held) at ttl/2. */
+    leaseRenewed(extra) {
+        this.emit("counter", WORKER_METRIC_NAMES.leaseRenewed, 1, extra);
+    }
+    /** The worker lost its lease (renew CAS failed / taken over / errored). */
+    leaseLost(extra) {
+        this.emit("counter", WORKER_METRIC_NAMES.leaseLost, 1, extra);
+    }
+    /** Snapshot the current consumer lag in seconds. */
+    recordLag(seconds, extra) {
+        this.emit("gauge", WORKER_METRIC_NAMES.lag, seconds, extra);
+    }
+    /**
+     * Snapshot the current durable HWM as a numeric position. A Redis-Streams id
+     * (`<ms>-<seq>`) is reduced to its leading ms component so the gauge is a
+     * monotonic-ish stall indicator (the same reduction outbox-dispatcher uses
+     * for `outbox.hwm`); callers with a numeric cursor pass it directly.
+     */
+    recordHwm(position, extra) {
+        this.emit("gauge", WORKER_METRIC_NAMES.currentHwm, position, extra);
+    }
+    /** Snapshot the current backlog / stream depth (pending entries). */
+    recordDepth(depth, extra) {
+        this.emit("gauge", WORKER_METRIC_NAMES.depth, depth, extra);
+    }
+    /** Record the wall-clock ms a graceful drain (stop) took. */
+    recordDrainDuration(ms, extra) {
+        this.emit("histogram", WORKER_METRIC_NAMES.drainDurationMs, ms, extra);
+    }
+    /** Count rows removed by a sweeper retention sweep. */
+    recordSwept(count, extra) {
+        this.emit("counter", WORKER_METRIC_NAMES.swept, count, extra);
+    }
+}
+/**
+ * Reduce a Redis-Streams id (`<ms>-<seq>`) to a numeric gauge position (its
+ * leading ms component). Mirrors outbox-dispatcher's `hwmGaugeValue`. A
+ * non-streams numeric string returns its Number; an unparseable value returns 0.
+ */
+export function hwmGaugeValue(streamId) {
+    if (!streamId)
+        return 0;
+    const dash = streamId.indexOf("-");
+    const head = dash < 0 ? streamId : streamId.slice(0, dash);
+    const n = Number(head);
+    return Number.isFinite(n) ? n : 0;
+}
+//# sourceMappingURL=observability-hooks.js.map

package/dist/worker/observability-hooks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"observability-hooks.js","sourceRoot":"","sources":["../../src/worker/observability-hooks.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,4EAA4E;AAC5E,EAAE;AACF,iFAAiF;AACjF,kEAAkE;AAClE,oDAAoD;AACpD,oDAAoD;AACpD,oDAAoD;AACpD,6DAA6D;AAC7D,mDAAmD;AACnD,0EAA0E;AAC1E,2DAA2D;AAC3D,qEAAqE;AACrE,EAAE;AACF,4EAA4E;AAC5E,+EAA+E;AAC/E,iFAAiF;AACjF,yEAAyE;AACzE,6EAA6E;AAC7E,EAAE;AACF,gFAAgF;AAChF,+EAA+E;AAC/E,6EAA6E;AAC7E,+EAA+E;AAC/E,0EAA0E;AAC1E,iFAAiF;AACjF,8EAA8E;AAE9E,OAAO,EAAyB,YAAY,EAAE,MAAM,eAAe,CAAC;AAGpE,0EAA0E;AAC1E,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,aAAa,EAAE,mCAAmC;IAClD,YAAY,EAAE,kCAAkC;IAChD,SAAS,EAAE,+BAA+B;IAC1C,GAAG,EAAE,kBAAkB;IACvB,UAAU,EAAE,0BAA0B;IACtC,KAAK,EAAE,oBAAoB;IAC3B,eAAe,EAAE,gCAAgC;IACjD,KAAK,EAAE,0BAA0B;CACzB,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAA6B;IACnE,CAAC,mBAAmB,CAAC,aAAa,CAAC,EAAE;QACnC,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,SAAS;QACf,WAAW,EACT,2EAA2E;QAC7E,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,YAAY,CAAC,EAAE;QAClC,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,SAAS;QACf,WAAW,EAAE,mDAAmD;QAChE,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,SAAS,CAAC,EAAE;QAC/B,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,SAAS;QACf,WAAW,EACT,6EAA6E;QAC/E,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,GAAG,CAAC,EAAE;QACzB,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,SAAS;QACf,WAAW,EACT,2EAA2E;QAC7E,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE;QAChC,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,OAAO;QACb,WAAW,EACT,0FAA0F;QAC5F,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE;QAC3B,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,OAAO;QACb,WAAW,EACT,sEAAsE;QACxE,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,eAAe,CAAC,EAAE;QACrC,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,IAAI;QACV,WAAW,EACT,kFAAkF;QACpF,QAAQ,EAAE,MAAM;KACjB;IACD,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE;QAC3B,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,SAAS;QACf,WAAW,EAAE,kDAAkD;QAC/D,QAAQ,EAAE,MAAM;KACjB;CACF,CAAC;AAmCF,kFAAkF;AAClF,SAAS,eAAe,CACtB,KAA6B,EAC7B,KAAuC;IAEvC,MAAM,GAAG,GAAoC,EAAE,CAAC;IAChD,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,SAAS;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,kBAAkB;IACZ,KAAK,CAAyB;IAC9B,MAAM,CAAiB;IAExC,YAAY,IAA4B;QACtC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC;QAC7B,8DAA8D;QAC9D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC;IAC5C,CAAC;IAEO,IAAI,CACV,IAAgB,EAChB,IAAY,EACZ,KAAa,EACb,KAAuC;QAEvC,IAAI,CAAC,MAAM,CAAC;YACV,IAAI;YACJ,IAAI;YACJ,KAAK;YACL,UAAU,EAAE,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,mDAAmD;IACnD,aAAa,CAAC,KAAuC;QACnD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,aAAa,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,wDAAwD;IACxD,YAAY,CAAC,KAAuC;QAClD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,YAAY,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IACnE,CAAC;IAED,2EAA2E;IAC3E,SAAS,CAAC,KAAuC;QAC/C,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,SAAS,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IAChE,CAAC;IAED,oDAAoD;IACpD,SAAS,CAAC,OAAe,EAAE,KAAuC;QAChE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,QAAgB,EAAE,KAAuC;QACjE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACtE,CAAC;IAED,qEAAqE;IACrE,WAAW,CAAC,KAAa,EAAE,KAAuC;QAChE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,6DAA6D;IAC7D,mBAAmB,CACjB,EAAU,EACV,KAAuC;QAEvC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,mBAAmB,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;IACzE,CAAC;IAED,uDAAuD;IACvD,WAAW,CAAC,KAAa,EAAE,KAAuC;QAChE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAChE,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,QAAuB;IACnD,IAAI,CAAC,QAAQ;QAAE,OAAO,CAAC,CAAC;IACxB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC3D,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IACvB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC"}

package/dist/worker/outbox-dispatcher.d.ts

@@ -0,0 +1,106 @@
+import type { CheckpointStore } from "./checkpoint-store.js";
+import type { DeadLetterSink } from "./dead-letter.js";
+import { ObservabilityHooks } from "./observability-hooks.js";
+import { type SingleActiveLeaseOpts } from "./single-active-lease.js";
+import type { SqlExecutor } from "./tx.js";
+import type { WorkerHandle, WorkerHwm, WorkerStopOptions, WorkerStopResult } from "./worker-handle.js";
+/** A pending outbox row claimed for dispatch. The generic shape the drain needs. */
+export interface OutboxRow {
+    /** Primary key (used to mark dispatched). */
+    id: string | number;
+    /** The dedupe id the consumer dedupes on. */
+    event_id: string;
+    /** The outbox topic. */
+    topic: string;
+    /** Source emission time (ms epoch or ISO) for lag + FIFO order. */
+    occurred_at: string | number;
+    /** The rest of the row (the publisher serializes whatever it needs). */
+    [key: string]: unknown;
+}
+/** The injected publisher — XADDs (or otherwise emits) a row + returns the id. */
+export type OutboxPublisher = (row: OutboxRow) => Promise<string | null>;
+export interface OutboxDispatcherOpts {
+    /** The injected publisher (e.g. an XADD to the topic's stream). */
+    publish: OutboxPublisher;
+    /**
+     * Opens a claim transaction. The drain claims rows + marks dispatched +
+     * advances the cursor on THIS tx so the mark + cursor commit together-or-none.
+     */
+    begin: <T>(fn: (tx: SqlExecutor) => Promise<T>) => Promise<T>;
+    /** Claims a batch of pending rows ON the claim tx (FOR UPDATE SKIP LOCKED). */
+    claimBatch: (tx: SqlExecutor, batchSize: number) => Promise<OutboxRow[]>;
+    /** Marks a row dispatched ON the claim tx (sets dispatched_at + stream_id). */
+    markDispatched: (tx: SqlExecutor, id: string | number, streamId: string | null) => Promise<void>;
+    /** Records a publish failure ON the claim tx (D401: bump attempts; keep row). */
+    recordFailure?: (tx: SqlExecutor, id: string | number, errorMessage: string) => Promise<void>;
+    checkpoint: CheckpointStore;
+    /** Lease config (service + scope=outbox + ttlMs). */
+    lease: Omit<SingleActiveLeaseOpts, "onLeaseLost">;
+    /** Checkpoint subscription key. Default `outbox:<service>`. */
+    subscription?: string;
+    /**
+     * OPTIONAL and currently NOT invoked on any drain path. Per D401 the producer
+     * drain has NO source-side dead-letter: a terminal / unpublishable row is
+     * surfaced via `recordFailure` (keep-row, publish_attempts bumped) + the
+     * 24h-unpublished sweep, NOT a sink. The sink is accepted for API symmetry with
+     * the consumer workers (StreamsWorker/ReplicationWorker, which DO dead-letter
+     * poison events) and a future opt-in; if provided it is PII-redacted, but no
+     * path calls it today.
+     */
+    deadLetter?: DeadLetterSink;
+    /** Rows claimed per drain pass. Default 100. */
+    batchSize?: number;
+    /** Poll interval (ms) when the loop runs autonomously. Default 1_000. */
+    pollMs?: number;
+    hooks?: ObservabilityHooks;
+    workerLabel?: string;
+    /** Launch the background drain loop on `start()`. Default true; tests pass
+     *  false to drive `drainOnce()`/`runOnce()` deterministically. */
+    autoStart?: boolean;
+}
+/**
+ * OutboxDispatcher — the Pattern-3 producer drain. Composes lease + checkpoint +
+ * hooks + drain. Single-active; the mark-dispatched + cursor advance are atomic
+ * in the claim tx; publish is at-least-once (the consumer dedupes).
+ */
+export declare class OutboxDispatcher implements WorkerHandle {
+    private readonly publish;
+    private readonly begin;
+    private readonly claimBatch;
+    private readonly markDispatched;
+    private readonly recordFailure?;
+    private readonly checkpoint;
+    private readonly lease;
+    private readonly subscription;
+    private readonly deadLetter?;
+    private readonly batchSize;
+    private readonly pollMs;
+    private readonly hooks;
+    private readonly autoStart;
+    private _running;
+    private _stopped;
+    private _inFlight;
+    private _hwm;
+    private _lastAppliedAtMs;
+    private loop;
+    constructor(opts: OutboxDispatcherOpts);
+    get running(): boolean;
+    get leaseAcquired(): boolean;
+    get lag(): number | null;
+    get currentHwm(): WorkerHwm;
+    start(): Promise<void>;
+    private hydrateCursor;
+    private runLoop;
+    /**
+     * One drain pass: claim a batch in ONE tx, publish each, mark dispatched +
+     * advance the HWM cursor (fence-guarded) in the SAME tx. Returns the count
+     * dispatched. The publish runs INSIDE the claim tx so the row-lock is held
+     * across it (a second dispatcher can't double-claim); at-least-once safe (a
+     * crash after publish but before commit re-publishes; the consumer dedupes).
+     */
+    drainOnce(): Promise<number>;
+    /** Run exactly one drain pass (test seam / cron tick). */
+    runOnce(): Promise<number>;
+    stop(opts?: WorkerStopOptions): Promise<WorkerStopResult>;
+}
+//# sourceMappingURL=outbox-dispatcher.d.ts.map

package/dist/worker/outbox-dispatcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outbox-dispatcher.d.ts","sourceRoot":"","sources":["../../src/worker/outbox-dispatcher.ts"],"names":[],"mappings":"AA2CA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAiB,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC7E,OAAO,EAIL,KAAK,qBAAqB,EAC3B,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EACV,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,oBAAoB,CAAC;AAE5B,oFAAoF;AACpF,MAAM,WAAW,SAAS;IACxB,6CAA6C;IAC7C,EAAE,EAAE,MAAM,GAAG,MAAM,CAAC;IACpB,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,WAAW,EAAE,MAAM,GAAG,MAAM,CAAC;IAC7B,wEAAwE;IACxE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,kFAAkF;AAClF,MAAM,MAAM,eAAe,GAAG,CAAC,GAAG,EAAE,SAAS,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;AAEzE,MAAM,WAAW,oBAAoB;IACnC,mEAAmE;IACnE,OAAO,EAAE,eAAe,CAAC;IACzB;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9D,+EAA+E;IAC/E,UAAU,EAAE,CAAC,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACzE,+EAA+E;IAC/E,cAAc,EAAE,CACd,EAAE,EAAE,WAAW,EACf,EAAE,EAAE,MAAM,GAAG,MAAM,EACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,KACpB,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,iFAAiF;IACjF,aAAa,CAAC,EAAE,CACd,EAAE,EAAE,WAAW,EACf,EAAE,EAAE,MAAM,GAAG,MAAM,EACnB,YAAY,EAAE,MAAM,KACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,UAAU,EAAE,eAAe,CAAC;IAC5B,qDAAqD;IACrD,KAAK,EAAE,IAAI,CAAC,qBAAqB,EAAE,aAAa,CAAC,CAAC;IAClD,+DAA+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,kBAAkB,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;sEACkE;IAClE,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAMD;;;;GAIG;AACH,qBAAa,gBAAiB,YAAW,YAAY;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAEN;IAChB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAGD;IAC1B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAIZ;IACnB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAIZ;IACnB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAkB;IAC7C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoB;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAGtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAiB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IAEpC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,IAAI,CAAqC;IACjD,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,IAAI,CAAoC;gBAEpC,IAAI,EAAE,oBAAoB;IA6BtC,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,GAAG,IAAI,MAAM,GAAG,IAAI,CAGvB;IAED,IAAI,UAAU,IAAI,SAAS,CAE1B;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAWd,aAAa;YAWb,OAAO;IA2BrB;;;;;;OAMG;IACG,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;IA8ElC,0DAA0D;IACpD,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC;IAI1B,IAAI,CAAC,IAAI,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAsBhE"}

package/dist/worker/outbox-dispatcher.js

@@ -0,0 +1,285 @@
+// D402 — OutboxDispatcher: the Pattern-3 PRODUCER drain.
+//
+// D402 (LOCKED): "OutboxDispatcher — Pattern-3 producer drain." It GENERALIZES
+// the pattern shipped in `ts/outbox-dispatcher/src/drainer.ts` (which this lib
+// does NOT edit — outbox-dispatcher gets re-based onto this substrate in
+// Phase 2). The producer side of the both-sides-completeness rule: every outbox
+// topic has a standardized producer (this) AND a standardized consumer
+// (StreamsWorker / ReplicationWorker).
+//
+// ── Composition (which primitives, how) ──────────────────────────────────────
+//   - SingleActiveLease  — single-active per (service, outbox) so two dispatcher
+//     replicas don't both drain. ttl/2 renew inside the lease. `validateFence`
+//     guards the mark-dispatched write (a lease-lost dispatcher's late mark is
+//     rejected — it must not mark a row another dispatcher is publishing).
+//   - CheckpointStore    — tracks the HWM (the last published stream id) so the
+//     `nodii.worker.current_hwm` gauge + resume position are durable. The mark
+//     UPDATE and the cursor advance run in the SAME claim tx (atomic).
+//   - ObservabilityHooks — lease/hwm/depth/drain metrics, canonical names.
+//   - SignalDrain        — `stop({timeoutMs})` registerable; bounded drain.
+//   - DeadLetterSink     — OPTIONAL and currently NOT invoked on any path. Per
+//     D401 the producer drain has NO source-side dead-letter: a terminal /
+//     unpublishable row is surfaced via `recordFailure` (keep-row,
+//     publish_attempts bumped) + the 24h-unpublished sweep, NOT a sink. The sink
+//     is accepted for API symmetry with the consumer workers + a future opt-in
+//     (PII-redacted if provided), but no drain path calls it today.
+//
+// ── The drain sequence (per 01-comms § 9.2 / outbox-dispatcher § 5.2) ─────────
+//   acquire-lease → claim a batch (dispatched_at IS NULL, occurred_at order,
+//   FOR UPDATE SKIP LOCKED) inside a tx → publish each (injected publisher) →
+//   mark dispatched (fence-guarded) + advance the HWM cursor, all in the SAME
+//   tx → commit. D401: a publish failure increments attempts + leaves the row in
+//   place (never DLQ).
+//
+// ── At-least-once semantics (NOT exactly-once on the producer side) ───────────
+//   The PUBLISH (XADD) runs BEFORE `validateFence` — it is NOT fenced. A stale or
+//   retried dispatcher's publish therefore DOES land: the producer is
+//   AT-LEAST-ONCE (the same row may be re-published after a crash-before-commit
+//   or a lease handoff). The fence guards only the mark-dispatched UPDATE — it
+//   stops a lease-lost dispatcher from falsely recording a row as dispatched
+//   (which would suppress the legitimate successor's re-publish). END-TO-END
+//   exactly-once is achieved by the CONSUMER's `consumed_events` dedupe (D198),
+//   NOT by the fence preventing the publish.
+import { ensurePiiRedaction } from "./dead-letter.js";
+import { hwmGaugeValue, ObservabilityHooks } from "./observability-hooks.js";
+import { FenceCheckUnavailableError, FenceLostError, SingleActiveLease, } from "./single-active-lease.js";
+const DEFAULT_BATCH_SIZE = 100;
+const DEFAULT_POLL_MS = 1_000;
+const DEFAULT_STOP_TIMEOUT_MS = 30_000;
+/**
+ * OutboxDispatcher — the Pattern-3 producer drain. Composes lease + checkpoint +
+ * hooks + drain. Single-active; the mark-dispatched + cursor advance are atomic
+ * in the claim tx; publish is at-least-once (the consumer dedupes).
+ */
+export class OutboxDispatcher {
+    publish;
+    begin;
+    claimBatch;
+    markDispatched;
+    recordFailure;
+    checkpoint;
+    lease;
+    subscription;
+    // OPTIONAL + currently uninvoked (see the opts docstring). Kept for API
+    // symmetry / a future opt-in; PII-redacted if provided.
+    deadLetter;
+    batchSize;
+    pollMs;
+    hooks;
+    autoStart;
+    _running = false;
+    _stopped = false;
+    _inFlight = 0;
+    _hwm = { lastStreamId: null };
+    _lastAppliedAtMs = null;
+    loop = Promise.resolve();
+    constructor(opts) {
+        this.publish = opts.publish;
+        this.begin = opts.begin;
+        this.claimBatch = opts.claimBatch;
+        this.markDispatched = opts.markDispatched;
+        this.recordFailure = opts.recordFailure;
+        this.checkpoint = opts.checkpoint;
+        this.lease = new SingleActiveLease({
+            ...opts.lease,
+            onLeaseLost: () => this.hooks.leaseLost(),
+        });
+        this.subscription = opts.subscription ?? `outbox:${opts.lease.service}`;
+        this.deadLetter = opts.deadLetter
+            ? ensurePiiRedaction(opts.deadLetter)
+            : undefined;
+        this.batchSize = opts.batchSize ?? DEFAULT_BATCH_SIZE;
+        this.pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
+        this.autoStart = opts.autoStart !== false;
+        this.hooks =
+            opts.hooks ??
+                new ObservabilityHooks({
+                    attributes: {
+                        service: opts.lease.service,
+                        scope: opts.lease.scope,
+                        worker: opts.workerLabel ?? "outbox",
+                    },
+                });
+    }
+    get running() {
+        return this._running;
+    }
+    get leaseAcquired() {
+        return this.lease.leaseAcquired;
+    }
+    get lag() {
+        if (this._lastAppliedAtMs === null)
+            return null;
+        return Math.max(0, (Date.now() - this._lastAppliedAtMs) / 1000);
+    }
+    get currentHwm() {
+        return this._hwm;
+    }
+    async start() {
+        if (this._running)
+            return;
+        this._stopped = false;
+        const acquired = await this.lease.acquire();
+        if (acquired)
+            this.hooks.leaseAcquired();
+        await this.hydrateCursor();
+        this._running = true;
+        // Background drain loop. `autoStart:false` lets a test drive `drainOnce()`.
+        if (this.autoStart)
+            this.loop = this.runLoop();
+    }
+    async hydrateCursor() {
+        try {
+            await this.begin(async (tx) => {
+                const cp = await this.checkpoint.get(this.subscription, tx);
+                if (cp)
+                    this._hwm = { lastStreamId: cp.lastStreamId };
+            });
+        }
+        catch {
+            /* cold start — cursor stays null */
+        }
+    }
+    async runLoop() {
+        while (!this._stopped) {
+            try {
+                if (!this.lease.leaseAcquired) {
+                    const got = await this.lease.acquire();
+                    if (got)
+                        this.hooks.leaseAcquired();
+                    else {
+                        await new Promise((r) => setTimeout(r, this.pollMs));
+                        continue;
+                    }
+                }
+                const n = await this.drainOnce();
+                if (n < this.batchSize) {
+                    await new Promise((r) => setTimeout(r, this.pollMs));
+                }
+            }
+            catch (err) {
+                // A drain-level error (claim tx itself rejected — Postgres down) is the
+                // Pillar-5 infra-failure case: back off, NEVER crash the loop.
+                this.hooks.recordDepth(-1, { error: "drain_failed" });
+                void err;
+                await new Promise((r) => setTimeout(r, Math.min(this.pollMs, 200)));
+            }
+        }
+    }
+    /**
+     * One drain pass: claim a batch in ONE tx, publish each, mark dispatched +
+     * advance the HWM cursor (fence-guarded) in the SAME tx. Returns the count
+     * dispatched. The publish runs INSIDE the claim tx so the row-lock is held
+     * across it (a second dispatcher can't double-claim); at-least-once safe (a
+     * crash after publish but before commit re-publishes; the consumer dedupes).
+     */
+    async drainOnce() {
+        if (!this.lease.leaseAcquired) {
+            const got = await this.lease.acquire();
+            if (got)
+                this.hooks.leaseAcquired();
+            else
+                return 0;
+        }
+        const token = this.lease.fencingToken;
+        let dispatched = 0;
+        let lastStreamId = null;
+        await this.begin(async (tx) => {
+            const rows = await this.claimBatch(tx, this.batchSize);
+            if (rows.length === 0)
+                return;
+            for (const row of rows) {
+                if (this.lease.lost)
+                    break;
+                this._inFlight += 1;
+                let fenceLost = false;
+                try {
+                    const streamId = await this.publish(row);
+                    // The fence guard BEFORE marking the row dispatched: a lease-lost
+                    // dispatcher must not mark a row a successor is also publishing.
+                    if (token === null) {
+                        throw new FenceLostError({
+                            leaseKey: this.lease.key,
+                            expectedToken: -1,
+                            current: null,
+                        });
+                    }
+                    await this.lease.validateFence(token);
+                    await this.markDispatched(tx, row.id, streamId);
+                    if (streamId)
+                        lastStreamId = streamId;
+                    dispatched += 1;
+                    this._lastAppliedAtMs = Date.now();
+                }
+                catch (err) {
+                    if (err instanceof FenceLostError ||
+                        err instanceof FenceCheckUnavailableError) {
+                        // The lease was lost/taken-over (FenceLostError) OR the fence-check
+                        // READ failed transiently (FenceCheckUnavailableError — lease-redis
+                        // blip). Either way ABORT the rest of the batch WITHOUT recordFailure:
+                        // a transient lease-redis blip is NOT a publish failure, so it must
+                        // NOT bump publish_attempts (5 blips would otherwise silently abandon
+                        // a row whose every publish succeeded). The row is retried on lease
+                        // recovery; the tx commits only the rows marked BEFORE the loss (each
+                        // fence-validated). Mirrors Streams/Replication + the Go dispatcher.
+                        this.hooks.leaseLost();
+                        fenceLost = true;
+                    }
+                    else {
+                        // D401: a publish failure NEVER deletes / DLQs the row — record the
+                        // failure (bump attempts) + leave it. The 24h-unpublished sweep
+                        // surfaces persistent liveness problems.
+                        const msg = err instanceof Error ? err.message : String(err);
+                        if (this.recordFailure) {
+                            await this.recordFailure(tx, row.id, msg);
+                        }
+                    }
+                }
+                finally {
+                    this._inFlight -= 1;
+                }
+                if (fenceLost)
+                    break;
+            }
+            // Advance the HWM cursor in the SAME tx as the marks (atomic): the cursor
+            // can never advance past a row that wasn't actually marked dispatched.
+            if (lastStreamId !== null) {
+                await this.checkpoint.advance({ subscription: this.subscription, lastStreamId }, tx);
+            }
+        });
+        if (lastStreamId !== null) {
+            this._hwm = { lastStreamId };
+            this.hooks.recordHwm(hwmGaugeValue(lastStreamId));
+        }
+        this.hooks.recordDepth(dispatched);
+        return dispatched;
+    }
+    /** Run exactly one drain pass (test seam / cron tick). */
+    async runOnce() {
+        return this.drainOnce();
+    }
+    async stop(opts) {
+        const timeoutMs = opts?.timeoutMs ?? DEFAULT_STOP_TIMEOUT_MS;
+        const start = Date.now();
+        this._stopped = true;
+        this._running = false;
+        try {
+            await Promise.race([
+                this.loop,
+                new Promise((r) => setTimeout(r, timeoutMs)),
+            ]);
+        }
+        catch {
+            /* swallow */
+        }
+        while (this._inFlight > 0 && Date.now() - start < timeoutMs) {
+            await new Promise((r) => setTimeout(r, 10));
+        }
+        const drainedCleanly = this._inFlight === 0;
+        await this.lease.release();
+        const durationMs = Date.now() - start;
+        this.hooks.recordDrainDuration(durationMs);
+        return { drainedCleanly, durationMs };
+    }
+}
+//# sourceMappingURL=outbox-dispatcher.js.map

package/dist/worker/outbox-dispatcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"outbox-dispatcher.js","sourceRoot":"","sources":["../../src/worker/outbox-dispatcher.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,EAAE;AACF,+EAA+E;AAC/E,+EAA+E;AAC/E,yEAAyE;AACzE,gFAAgF;AAChF,uEAAuE;AACvE,uCAAuC;AACvC,EAAE;AACF,gFAAgF;AAChF,iFAAiF;AACjF,+EAA+E;AAC/E,+EAA+E;AAC/E,2EAA2E;AAC3E,gFAAgF;AAChF,+EAA+E;AAC/E,uEAAuE;AACvE,2EAA2E;AAC3E,4EAA4E;AAC5E,+EAA+E;AAC/E,2EAA2E;AAC3E,mEAAmE;AACnE,iFAAiF;AACjF,+EAA+E;AAC/E,oEAAoE;AACpE,EAAE;AACF,iFAAiF;AACjF,6EAA6E;AAC7E,8EAA8E;AAC9E,8EAA8E;AAC9E,iFAAiF;AACjF,uBAAuB;AACvB,EAAE;AACF,iFAAiF;AACjF,kFAAkF;AAClF,sEAAsE;AACtE,gFAAgF;AAChF,+EAA+E;AAC/E,6EAA6E;AAC7E,6EAA6E;AAC7E,gFAAgF;AAChF,6CAA6C;AAG7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC7E,OAAO,EACL,0BAA0B,EAC1B,cAAc,EACd,iBAAiB,GAElB,MAAM,0BAA0B,CAAC;AA0ElC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAEvC;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IACV,OAAO,CAAkB;IACzB,KAAK,CAEN;IACC,UAAU,CAGD;IACT,cAAc,CAIZ;IACF,aAAa,CAIX;IACF,UAAU,CAAkB;IAC5B,KAAK,CAAoB;IACzB,YAAY,CAAS;IACtC,wEAAwE;IACxE,wDAAwD;IACvC,UAAU,CAAkB;IAC5B,SAAS,CAAS;IAClB,MAAM,CAAS;IACf,KAAK,CAAqB;IAC1B,SAAS,CAAU;IAE5B,QAAQ,GAAG,KAAK,CAAC;IACjB,QAAQ,GAAG,KAAK,CAAC;IACjB,SAAS,GAAG,CAAC,CAAC;IACd,IAAI,GAAc,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACzC,gBAAgB,GAAkB,IAAI,CAAC;IACvC,IAAI,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEhD,YAAY,IAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;QACxC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,IAAI,iBAAiB,CAAC;YACjC,GAAG,IAAI,CAAC,KAAK;YACb,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;SAC1C,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,UAAU,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACxE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU;YAC/B,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC;YACrC,CAAC,CAAC,SAAS,CAAC;QACd,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;QACtD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,eAAe,CAAC;QAC7C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC;QAC1C,IAAI,CAAC,KAAK;YACR,IAAI,CAAC,KAAK;gBACV,IAAI,kBAAkB,CAAC;oBACrB,UAAU,EAAE;wBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;wBAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;wBACvB,MAAM,EAAE,IAAI,CAAC,WAAW,IAAI,QAAQ;qBACrC;iBACF,CAAC,CAAC;IACP,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;IAClC,CAAC;IAED,IAAI,GAAG;QACL,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC5C,IAAI,QAAQ;YAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;QACzC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,4EAA4E;QAC5E,IAAI,IAAI,CAAC,SAAS;YAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAC5B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAC5D,IAAI,EAAE;oBAAE,IAAI,CAAC,IAAI,GAAG,EAAE,YAAY,EAAE,EAAE,CAAC,YAAY,EAAE,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,oCAAoC;QACtC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;oBAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;oBACvC,IAAI,GAAG;wBAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;yBAC/B,CAAC;wBACJ,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;wBAC3D,SAAS;oBACX,CAAC;gBACH,CAAC;gBACD,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACvB,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,wEAAwE;gBACxE,+DAA+D;gBAC/D,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;gBACtD,KAAK,GAAG,CAAC;gBACT,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAC5B,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAC1C,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACvC,IAAI,GAAG;gBAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;;gBAC/B,OAAO,CAAC,CAAC;QAChB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC;QACtC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAkB,IAAI,CAAC;QACvC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YACvD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;oBAAE,MAAM;gBAC3B,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;gBACpB,IAAI,SAAS,GAAG,KAAK,CAAC;gBACtB,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBACzC,kEAAkE;oBAClE,iEAAiE;oBACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;wBACnB,MAAM,IAAI,cAAc,CAAC;4BACvB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG;4BACxB,aAAa,EAAE,CAAC,CAAC;4BACjB,OAAO,EAAE,IAAI;yBACd,CAAC,CAAC;oBACL,CAAC;oBACD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBACtC,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;oBAChD,IAAI,QAAQ;wBAAE,YAAY,GAAG,QAAQ,CAAC;oBACtC,UAAU,IAAI,CAAC,CAAC;oBAChB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACrC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IACE,GAAG,YAAY,cAAc;wBAC7B,GAAG,YAAY,0BAA0B,EACzC,CAAC;wBACD,oEAAoE;wBACpE,oEAAoE;wBACpE,uEAAuE;wBACvE,oEAAoE;wBACpE,sEAAsE;wBACtE,oEAAoE;wBACpE,sEAAsE;wBACtE,qEAAqE;wBACrE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;wBACvB,SAAS,GAAG,IAAI,CAAC;oBACnB,CAAC;yBAAM,CAAC;wBACN,oEAAoE;wBACpE,gEAAgE;wBAChE,yCAAyC;wBACzC,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBAC7D,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;4BACvB,MAAM,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;wBAC5C,CAAC;oBACH,CAAC;gBACH,CAAC;wBAAS,CAAC;oBACT,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;gBACtB,CAAC;gBACD,IAAI,SAAS;oBAAE,MAAM;YACvB,CAAC;YACD,0EAA0E;YAC1E,uEAAuE;YACvE,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3B,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,EACjD,EAAE,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,IAAI,GAAG,EAAE,YAAY,EAAE,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QACnC,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,0DAA0D;IAC1D,KAAK,CAAC,OAAO;QACX,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAwB;QACjC,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,uBAAuB,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjB,IAAI,CAAC,IAAI;gBACT,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;aACnD,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,aAAa;QACf,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;YAC5D,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC3C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC;IACxC,CAAC;CACF"}