@oneuptime/common 14.0.8 → 14.0.10

package/build/dist/Server/Utils/Telemetry/ProxmoxNativeNodeLiveness.js

@@ -0,0 +1,296 @@
+import GlobalCache from "../../Infrastructure/GlobalCache";
+import InMemoryTTLCache from "../../Infrastructure/InMemoryTTLCache";
+import logger from "../Logger";
+/*
+ * ------------------------------------------------------------------
+ * Proxmox VE native push — which nodes have stopped reporting
+ * ------------------------------------------------------------------
+ *
+ * With Proxmox VE's built-in OpenTelemetry push every node reports only
+ * itself, so a node that dies simply goes quiet: nothing ever says
+ * pve_up = 0 for it, and the Node Offline / Cluster Quorum at Risk
+ * templates could not fire. (The Proxmox Agent does not have this gap —
+ * pve-exporter asks the cluster about every node.)
+ *
+ * The nodes that are still alive know better, so they speak for the
+ * silent ones: when a live node's own status push is ingested, it also
+ * carries a "not reporting" report for every sibling that has gone
+ * quiet (ProxmoxNativePush.appendProxmoxSiblingReportsInPlace). This
+ * module decides WHICH siblings are silent, and whether this push may
+ * report at all.
+ *
+ * Liveness lives in Redis, one key per (project, cluster, node), written
+ * on every node-status push and expiring SILENCE_MS after the last one:
+ * "streakStartMs,lastPushMs" on OneUptime's own receive clock — never the
+ * node's clock, which can be skewed.
+ *
+ * A sibling Y is silent, from reporter P's point of view, only when BOTH
+ *   - Y's key is gone: OneUptime has processed no status push from Y for
+ *     SILENCE_MS, and
+ *   - Y's last inventory sighting is older than P's own push time minus
+ *     SILENCE_MS (both on the PVE clocks, which corosync keeps close).
+ *     This keeps a node that something else still reports — the agent,
+ *     in a cluster someone runs both ways — from being called silent,
+ *     and turns clock skew into a later report rather than a false one.
+ *
+ * Who reports: every live node, but only once the cluster has an
+ * ESTABLISHED node — one that has pushed CONTINUOUSLY (no gap over
+ * STREAK_GAP_MS) for at least SILENCE_MS. After a OneUptime ingest outage
+ * or backlog every key has expired, and the first node processed
+ * afterwards would see all its siblings as silent; by the time any node
+ * is established, every sibling that is alive has a key again. Having
+ * every live node report (not just the established ones) keeps Quorum at
+ * Risk at L ÷ (L + D): each report carries D ÷ L of weight, where L counts
+ * every node not being reported (a node is presumed live until it is
+ * reported), so however many pushes each live node lands in a minute the
+ * silent nodes weigh as much as live ones. A push that carries no report
+ * — a Redis error fails closed — lifts its minute above that.
+ *
+ * The established gate guards a node's FIRST report only. A node already
+ * reported down — Offline in the inventory, and marked within the
+ * monitors' window (MONITOR_WINDOW_MS) — keeps being reported by every
+ * live node even while none is established: after a OneUptime restart or
+ * a Redis failover that lost the keys, or a lone survivor's own gap or
+ * bursty processing. Otherwise the pushes in those two minutes would be
+ * stored with no report, their minutes would read 100 % up, and Node
+ * Offline and Quorum at Risk would resolve only to page again minutes
+ * later. The mark is its own column, notReportingMarkedAt, written only
+ * by ProxmoxResourceService.markNodesNotReporting on the ingest worker's
+ * clock (refreshed at most once a minute while the node stays reported)
+ * and cleared by the node's own next push — so it survives the loss of
+ * Redis, the continuing reports keep it fresh, and nothing else (an agent
+ * scrape, an adoption, a database clock) can pass for it. Its age is
+ * taken at the moment the roster was read, so every push served one
+ * cached roster decides alike. After a longer silence of the whole
+ * cluster — an outage beyond the monitors' window — the window holds
+ * nothing from before and the incidents have already resolved, so there
+ * is no continuity to keep, and a node that came back during the outage
+ * must not be reported until its own first push is processed: its first
+ * report waits for an established node again.
+ *
+ * False alarms this is built to avoid:
+ *   - A OneUptime ingest outage or backlog — see "Who reports" above.
+ *   - A whole cluster going dark (power, network, the metric server
+ *     removed): nobody is left to report, so nobody is paged per node —
+ *     the cluster turns Disconnected instead, exactly as when the agent
+ *     dies. A standalone host is the same case.
+ *   - Redis trouble: any error means no report (fail closed).
+ * ------------------------------------------------------------------
+ */
+// A node is silent after this long without a processed status push.
+export const PROXMOX_NODE_SILENCE_MS = 120000;
+/*
+ * The rolling window of the Node Offline and Cluster Quorum at Risk
+ * templates. Reports of nodes already Offline continue through a gap only
+ * while they were marked within it (see the header).
+ */
+export const PROXMOX_MONITOR_WINDOW_MS = 5 * 60000;
+/*
+ * A reporter's push streak restarts when two of its pushes are further
+ * apart than this. pvestatd pushes every ~10 s; a gap of a minute means
+ * the pushes (or OneUptime's processing of them) were interrupted.
+ */
+export const PROXMOX_NODE_STREAK_GAP_MS = 60000;
+const LIVENESS_NAMESPACE = "proxmox-native-node-live";
+// How long an ingest worker reuses a cluster's node roster.
+export const PROXMOX_ROSTER_CACHE_TTL_MS = 30000;
+export function parseProxmoxNodeLiveness(value) {
+    if (!value) {
+        return null;
+    }
+    const [startRaw, lastRaw] = value.split(",");
+    // Number("") is 0 — an empty field must not read as the epoch.
+    if (!startRaw || !lastRaw || !startRaw.trim() || !lastRaw.trim()) {
+        return null;
+    }
+    const streakStartMs = Number(startRaw);
+    const lastPushMs = Number(lastRaw);
+    if (!Number.isFinite(streakStartMs) ||
+        !Number.isFinite(lastPushMs) ||
+        streakStartMs > lastPushMs) {
+        return null;
+    }
+    return { streakStartMs, lastPushMs };
+}
+export function serializeProxmoxNodeLiveness(liveness) {
+    return `${liveness.streakStartMs},${liveness.lastPushMs}`;
+}
+/*
+ * The liveness after a push processed at nowMs. The streak survives
+ * only if the previous push was recent enough; a push processed out of
+ * order (older than the last one) never moves lastPushMs back.
+ */
+export function nextProxmoxNodeLiveness(previous, nowMs) {
+    if (!previous || nowMs - previous.lastPushMs > PROXMOX_NODE_STREAK_GAP_MS) {
+        return { streakStartMs: nowMs, lastPushMs: nowMs };
+    }
+    return {
+        streakStartMs: previous.streakStartMs,
+        lastPushMs: Math.max(previous.lastPushMs, nowMs),
+    };
+}
+// A status push from this node was processed within the silence window.
+export function isAliveProxmoxNode(liveness, nowMs) {
+    return Boolean(liveness && nowMs - liveness.lastPushMs <= PROXMOX_NODE_SILENCE_MS);
+}
+/*
+ * Established: has pushed continuously for at least the silence window,
+ * and the streak is still unbroken now — its last push is at most
+ * STREAK_GAP_MS old. The cluster reports silent nodes only while at least
+ * one node is established. The second condition matters right after a
+ * OneUptime outage a little shorter than the silence window: a key
+ * written before the outage is still alive, but that node's streak is
+ * already broken (its next push will restart it), so it must not vouch
+ * for the siblings whose keys expired a moment earlier.
+ */
+export function isEligibleProxmoxReporter(liveness, nowMs) {
+    return Boolean(liveness &&
+        nowMs - liveness.lastPushMs <= PROXMOX_NODE_STREAK_GAP_MS &&
+        nowMs - liveness.streakStartMs >= PROXMOX_NODE_SILENCE_MS);
+}
+/*
+ * Which siblings the reporter should report as not reporting, or null
+ * when it should report nothing. Pure — every input is explicit.
+ *
+ * Silent: not the reporter, not alive, and last seen more than the
+ * silence window before the reporter's own push. With an established
+ * node in the cluster every silent node is reported; without one, only
+ * those already Offline in the inventory and marked within the monitors'
+ * window of the time the roster was read (see the header).
+ */
+export function decideProxmoxSilentNodes(data) {
+    var _a;
+    const selfLiveness = data.liveness.get(data.selfNode) || null;
+    if (!isAliveProxmoxNode(selfLiveness, data.nowMs)) {
+        return null;
+    }
+    const nodes = new Map();
+    for (const node of data.roster) {
+        nodes.set(node.nodeName, node);
+    }
+    if (!nodes.has(data.selfNode)) {
+        // The reporter's own first row may not be in the inventory yet.
+        nodes.set(data.selfNode, null);
+    }
+    // A standalone host has nobody to speak for it.
+    if (nodes.size < 2) {
+        return null;
+    }
+    const silentBeforeMs = data.reporterTimeMs - PROXMOX_NODE_SILENCE_MS;
+    const silentNodes = [];
+    let established = false;
+    for (const [nodeName, rosterNode] of nodes) {
+        const liveness = data.liveness.get(nodeName) || null;
+        if (isEligibleProxmoxReporter(liveness, data.nowMs)) {
+            established = true;
+        }
+        if (nodeName !== data.selfNode &&
+            !isAliveProxmoxNode(liveness, data.nowMs) &&
+            rosterNode &&
+            rosterNode.lastSeenAt.getTime() < silentBeforeMs) {
+            silentNodes.push(nodeName);
+        }
+    }
+    let reported = silentNodes;
+    if (!established) {
+        const rosterReadAtMs = (_a = data.rosterReadAtMs) !== null && _a !== void 0 ? _a : data.nowMs;
+        reported = silentNodes.filter((nodeName) => {
+            const rosterNode = nodes.get(nodeName);
+            return Boolean(rosterNode &&
+                rosterNode.isUp === false &&
+                rosterNode.notReportingMarkedAt &&
+                rosterReadAtMs - rosterNode.notReportingMarkedAt.getTime() <=
+                    PROXMOX_MONITOR_WINDOW_MS);
+        });
+    }
+    if (reported.length === 0) {
+        return null;
+    }
+    reported.sort();
+    /*
+     * Every node not reported is presumed live and reports too — including
+     * a live sibling whose first push after a gap is not processed yet (it
+     * has no key, but is not reported either). Counting only the nodes with
+     * a key would hand the first node back all of the silent nodes' weight.
+     */
+    return {
+        silentNodes: reported,
+        reporterCount: nodes.size - reported.length,
+    };
+}
+export function isProxmoxSilentNodeDetectionEnabled() {
+    const raw = process.env["PVE_NATIVE_NODE_SILENCE_DETECTION"];
+    return !raw || raw.trim().toLowerCase() !== "false";
+}
+const rosterCache = new InMemoryTTLCache(5000);
+function livenessKey(projectId, proxmoxClusterId, nodeName) {
+    return `${projectId.toString()}:${proxmoxClusterId.toString()}:${nodeName}`;
+}
+/*
+ * Record this node-status push and decide which siblings it reports.
+ * Never throws: any failure reports nothing, and the push itself is
+ * ingested as usual.
+ */
+export async function recordProxmoxNodePushAndFindSilentNodes(data) {
+    var _a;
+    const nowMs = (_a = data.nowMs) !== null && _a !== void 0 ? _a : Date.now();
+    try {
+        const selfKey = livenessKey(data.projectId, data.proxmoxClusterId, data.selfNode);
+        const previous = parseProxmoxNodeLiveness(await GlobalCache.getString(LIVENESS_NAMESPACE, selfKey));
+        /*
+         * GET-then-SET: two workers processing this node's pushes at once can
+         * both miss the key and both start a streak. That only ever moves the
+         * streak start later — the conservative direction.
+         */
+        const current = nextProxmoxNodeLiveness(previous, nowMs);
+        await GlobalCache.setString(LIVENESS_NAMESPACE, selfKey, serializeProxmoxNodeLiveness(current), { expiresInSeconds: Math.ceil(PROXMOX_NODE_SILENCE_MS / 1000) });
+        const rosterKey = `${data.projectId.toString()}:${data.proxmoxClusterId.toString()}`;
+        let cached = rosterCache.get(rosterKey);
+        if (!cached) {
+            cached = {
+                roster: await data.loadRoster({
+                    projectId: data.projectId,
+                    proxmoxClusterId: data.proxmoxClusterId,
+                }),
+                readAtMs: nowMs,
+            };
+            rosterCache.set(rosterKey, cached, PROXMOX_ROSTER_CACHE_TTL_MS);
+        }
+        const roster = cached.roster;
+        const siblings = roster
+            .map((node) => {
+            return node.nodeName;
+        })
+            .filter((nodeName) => {
+            return nodeName !== data.selfNode;
+        });
+        if (siblings.length === 0) {
+            return null;
+        }
+        const values = await GlobalCache.getStrings(LIVENESS_NAMESPACE, siblings.map((nodeName) => {
+            return livenessKey(data.projectId, data.proxmoxClusterId, nodeName);
+        }));
+        const liveness = new Map();
+        liveness.set(data.selfNode, current);
+        siblings.forEach((nodeName, index) => {
+            liveness.set(nodeName, parseProxmoxNodeLiveness(values[index]));
+        });
+        return decideProxmoxSilentNodes({
+            selfNode: data.selfNode,
+            reporterTimeMs: data.reporterTimeMs,
+            nowMs,
+            roster,
+            liveness,
+            rosterReadAtMs: cached.readAtMs,
+        });
+    }
+    catch (err) {
+        logger.warn(`Proxmox node liveness check failed for cluster ${data.proxmoxClusterId.toString()} (no silent-node report this push): ${err instanceof Error ? err.message : String(err)}`);
+        return null;
+    }
+}
+// Test seam: drop cached rosters.
+export function clearProxmoxNodeRosterCache() {
+    rosterCache.clear();
+}
+//# sourceMappingURL=ProxmoxNativeNodeLiveness.js.map

package/build/dist/Server/Utils/Telemetry/ProxmoxNativeNodeLiveness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ProxmoxNativeNodeLiveness.js","sourceRoot":"","sources":["../../../../../Server/Utils/Telemetry/ProxmoxNativeNodeLiveness.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAC3D,OAAO,gBAAgB,MAAM,uCAAuC,CAAC;AAErE,OAAO,MAAM,MAAM,WAAW,CAAC;AAE/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2EG;AAEH,oEAAoE;AACpE,MAAM,CAAC,MAAM,uBAAuB,GAAW,MAAO,CAAC;AAEvD;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAW,CAAC,GAAG,KAAM,CAAC;AAE5D;;;;GAIG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAW,KAAM,CAAC;AAEzD,MAAM,kBAAkB,GAAW,0BAA0B,CAAC;AAE9D,4DAA4D;AAC5D,MAAM,CAAC,MAAM,2BAA2B,GAAW,KAAM,CAAC;AA6C1D,MAAM,UAAU,wBAAwB,CACtC,KAAgC;IAEhC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7C,+DAA+D;IAC/D,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QACjE,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,aAAa,GAAW,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAW,MAAM,CAAC,OAAO,CAAC,CAAC;IAC3C,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;QAC/B,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;QAC5B,aAAa,GAAG,UAAU,EAC1B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,QAA6B;IAE7B,OAAO,GAAG,QAAQ,CAAC,aAAa,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;AAC5D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CACrC,QAAoC,EACpC,KAAa;IAEb,IAAI,CAAC,QAAQ,IAAI,KAAK,GAAG,QAAQ,CAAC,UAAU,GAAG,0BAA0B,EAAE,CAAC;QAC1E,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IACrD,CAAC;IACD,OAAO;QACL,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,kBAAkB,CAChC,QAAoC,EACpC,KAAa;IAEb,OAAO,OAAO,CACZ,QAAQ,IAAI,KAAK,GAAG,QAAQ,CAAC,UAAU,IAAI,uBAAuB,CACnE,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,yBAAyB,CACvC,QAAoC,EACpC,KAAa;IAEb,OAAO,OAAO,CACZ,QAAQ;QACN,KAAK,GAAG,QAAQ,CAAC,UAAU,IAAI,0BAA0B;QACzD,KAAK,GAAG,QAAQ,CAAC,aAAa,IAAI,uBAAuB,CAC5D,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAaxC;;IACC,MAAM,YAAY,GAChB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IAC3C,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAA0C,IAAI,GAAG,EAAE,CAAC;IAC/D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,gEAAgE;QAChE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACjC,CAAC;IACD,gDAAgD;IAChD,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,cAAc,GAAW,IAAI,CAAC,cAAc,GAAG,uBAAuB,CAAC;IAC7E,MAAM,WAAW,GAAkB,EAAE,CAAC;IACtC,IAAI,WAAW,GAAY,KAAK,CAAC;IAEjC,KAAK,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,KAAK,EAAE,CAAC;QAC3C,MAAM,QAAQ,GACZ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACtC,IAAI,yBAAyB,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACpD,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;QACD,IACE,QAAQ,KAAK,IAAI,CAAC,QAAQ;YAC1B,CAAC,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC;YACzC,UAAU;YACV,UAAU,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,cAAc,EAChD,CAAC;YACD,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,GAAkB,WAAW,CAAC;IAC1C,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,cAAc,GAAW,MAAA,IAAI,CAAC,cAAc,mCAAI,IAAI,CAAC,KAAK,CAAC;QACjE,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,QAAgB,EAAE,EAAE;YACjD,MAAM,UAAU,GACd,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtB,OAAO,OAAO,CACZ,UAAU;gBACR,UAAU,CAAC,IAAI,KAAK,KAAK;gBACzB,UAAU,CAAC,oBAAoB;gBAC/B,cAAc,GAAG,UAAU,CAAC,oBAAoB,CAAC,OAAO,EAAE;oBACxD,yBAAyB,CAC9B,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB;;;;;OAKG;IACH,OAAO;QACL,WAAW,EAAE,QAAQ;QACrB,aAAa,EAAE,KAAK,CAAC,IAAI,GAAG,QAAQ,CAAC,MAAM;KAC5C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mCAAmC;IACjD,MAAM,GAAG,GACP,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC;AACtD,CAAC;AAOD,MAAM,WAAW,GAGZ,IAAI,gBAAgB,CAGtB,IAAK,CAAC,CAAC;AAEV,SAAS,WAAW,CAClB,SAAmB,EACnB,gBAA0B,EAC1B,QAAgB;IAEhB,OAAO,GAAG,SAAS,CAAC,QAAQ,EAAE,IAAI,gBAAgB,CAAC,QAAQ,EAAE,IAAI,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,uCAAuC,CAAC,IAO7D;;IACC,MAAM,KAAK,GAAW,MAAA,IAAI,CAAC,KAAK,mCAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IAC/C,IAAI,CAAC;QACH,MAAM,OAAO,GAAW,WAAW,CACjC,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,QAAQ,CACd,CAAC;QACF,MAAM,QAAQ,GAA+B,wBAAwB,CACnE,MAAM,WAAW,CAAC,SAAS,CAAC,kBAAkB,EAAE,OAAO,CAAC,CACzD,CAAC;QACF;;;;WAIG;QACH,MAAM,OAAO,GAAwB,uBAAuB,CAC1D,QAAQ,EACR,KAAK,CACN,CAAC;QACF,MAAM,WAAW,CAAC,SAAS,CACzB,kBAAkB,EAClB,OAAO,EACP,4BAA4B,CAAC,OAAO,CAAC,EACrC,EAAE,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,EAAE,CAChE,CAAC;QAEF,MAAM,SAAS,GAAW,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC7F,IAAI,MAAM,GAEM,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG;gBACP,MAAM,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC;oBAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;iBACxC,CAAC;gBACF,QAAQ,EAAE,KAAK;aAChB,CAAC;YACF,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,2BAA2B,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,MAAM,GAA6B,MAAM,CAAC,MAAM,CAAC;QAEvD,MAAM,QAAQ,GAAkB,MAAM;aACnC,GAAG,CAAC,CAAC,IAAuB,EAAE,EAAE;YAC/B,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,QAAgB,EAAE,EAAE;YAC3B,OAAO,QAAQ,KAAK,IAAI,CAAC,QAAQ,CAAC;QACpC,CAAC,CAAC,CAAC;QACL,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAyB,MAAM,WAAW,CAAC,UAAU,CAC/D,kBAAkB,EAClB,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAgB,EAAE,EAAE;YAChC,OAAO,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QACtE,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,QAAQ,GAA4C,IAAI,GAAG,EAAE,CAAC;QACpE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrC,QAAQ,CAAC,OAAO,CAAC,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE;YACnD,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,wBAAwB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,OAAO,wBAAwB,CAAC;YAC9B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,KAAK;YACL,MAAM;YACN,QAAQ;YACR,cAAc,EAAE,MAAM,CAAC,QAAQ;SAChC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CACT,kDAAkD,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,uCAAuC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC5K,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,kCAAkC;AAClC,MAAM,UAAU,2BAA2B;IACzC,WAAW,CAAC,KAAK,EAAE,CAAC;AACtB,CAAC"}