@oneuptime/common 13.0.3 → 13.0.5

package/build/dist/Server/Utils/SecurityEvent/Connectors/ConnectorPlatformHealth.js

@@ -0,0 +1,352 @@
+import OneUptimeDate from "../../../../Types/Date";
+import { ClickhouseAppInstance } from "../../../Infrastructure/ClickhouseDatabase";
+import Queue, { QueueName } from "../../../Infrastructure/Queue";
+import logger from "../../Logger";
+import { makeCheck } from "./Types";
+/*
+ * What the "Test connection" checklist can say about OneUptime itself.
+ *
+ * The connector complaint this exists for reads "connected, but nothing is
+ * ingested", and half the time the source is fine: the poll never ran
+ * because no process consumes the Worker queue (DISABLE_QUEUE_WORKERS=true
+ * on the app with no worker deployment), or the Workers feature set never
+ * registered the scheduler. Those facts live in Redis, not in the
+ * connection row, so a connection that was never polled looks exactly like
+ * a healthy one that found nothing. These probes read them out.
+ *
+ * Every probe is best-effort and independent: an unreachable Redis makes a
+ * value null ("unknown"), never a thrown error and never a fake pass.
+ */
+/*
+ * The job names the two poll crons are registered under. Both connector
+ * families' schedulers are listed so the check reads "the poll scheduler"
+ * whichever family asked.
+ *
+ * These are the RAW names, colon included, because that is what BullMQ
+ * reports. RunCron goes through Queue.addJob, which calls
+ * queue.add(jobName, {}, { jobId, repeat: { pattern, jobId } }): BullMQ's
+ * legacy repeatable API, not upsertJobScheduler. BullMQ 5.x
+ * (Repeat.updateRepeatableJob, addRepeatableJob-2.lua) stores that
+ * repeatable under an md5 of "name:jobId:endDate:tz:pattern" and writes
+ * only `name` and `pattern` into its metadata hash, so getJobSchedulers()
+ * lists it as { key: "<md5>", name: "SecurityEvents:Poll...", next,
+ * pattern } with no `id`. Matching the sanitized "SecurityEvents-Poll..."
+ * id against `id` therefore failed the scheduler check on every healthy
+ * install (review finding scheduler-check-always-fails).
+ */
+export const CONNECTOR_SCHEDULER_JOB_NAMES = [
+    "SecurityEvents:PollGoogleSecOpsConnections",
+    "SecurityEvents:PollSecurityEventConnections",
+];
+// Two scheduler ticks of grace before a poll counts as overdue.
+export const CONNECTOR_OVERDUE_GRACE_IN_MS = 2 * 60 * 1000;
+export default class ConnectorPlatformHealth {
+    static async getPlatformStatus(data) {
+        const status = {
+            workerConsumers: null,
+            schedulerRegistered: null,
+            queueWaiting: null,
+            queueFailed: null,
+            storageReachable: null,
+        };
+        let queue = null;
+        try {
+            queue =
+                (data === null || data === void 0 ? void 0 : data.queueOverride) ||
+                    Queue.getQueue(QueueName.Worker);
+        }
+        catch (error) {
+            logger.error("ConnectorPlatformHealth: the Worker queue could not be opened.");
+            logger.error(error);
+        }
+        if (queue) {
+            status.workerConsumers = await this.probeWorkerCount(queue);
+            try {
+                if (typeof queue.getJobSchedulers === "function") {
+                    const schedulers = await queue.getJobSchedulers(0, 1000, true);
+                    const matches = [];
+                    for (const scheduler of schedulers) {
+                        if (scheduler && this.isConnectorScheduler(scheduler)) {
+                            matches.push(scheduler);
+                        }
+                    }
+                    status.schedulerRegistered = matches.length > 0;
+                    const nextRuns = matches
+                        .map((scheduler) => {
+                        return typeof scheduler.next === "number" &&
+                            Number.isFinite(scheduler.next)
+                            ? scheduler.next
+                            : 0;
+                    })
+                        .filter((value) => {
+                        return value > 0;
+                    });
+                    if (nextRuns.length > 0) {
+                        status.schedulerNextRunAt = new Date(Math.min(...nextRuns)).toISOString();
+                    }
+                }
+            }
+            catch (error) {
+                logger.error("ConnectorPlatformHealth: scheduler probe failed.");
+                logger.error(error);
+            }
+            try {
+                status.queueWaiting = await queue.getWaitingCount();
+            }
+            catch (error) {
+                logger.error("ConnectorPlatformHealth: waiting count probe failed.");
+                logger.error(error);
+            }
+            try {
+                status.queueFailed = await queue.getFailedCount();
+            }
+            catch (error) {
+                logger.error("ConnectorPlatformHealth: failed count probe failed.");
+                logger.error(error);
+            }
+        }
+        try {
+            status.storageReachable = (data === null || data === void 0 ? void 0 : data.storageProbeOverride)
+                ? await data.storageProbeOverride()
+                : await ClickhouseAppInstance.checkConnnectionStatus();
+        }
+        catch (error) {
+            logger.error("ConnectorPlatformHealth: storage probe failed.");
+            logger.error(error);
+            status.storageReachable = false;
+        }
+        return status;
+    }
+    /*
+     * Whether one getJobSchedulers() entry is a connector poll cron. `name`
+     * is the match for every registration BullMQ can describe (see
+     * CONNECTOR_SCHEDULER_JOB_NAMES). The key prefix covers the one shape
+     * where the name cannot be read back: a repeatable still kept under a
+     * pre-md5 concatenated key ("name:jobId:endDate:tz:pattern") whose
+     * metadata hash is gone. BullMQ then rebuilds the entry by splitting the
+     * key on ":", which cuts "SecurityEvents:Poll..." in half and reports the
+     * name as "SecurityEvents", but the key still starts with the full name.
+     */
+    static isConnectorScheduler(scheduler) {
+        const name = typeof scheduler.name === "string" ? scheduler.name : "";
+        const key = typeof scheduler.key === "string" ? scheduler.key : "";
+        return CONNECTOR_SCHEDULER_JOB_NAMES.some((jobName) => {
+            return name === jobName || key.startsWith(`${jobName}:`);
+        });
+    }
+    static async probeWorkerCount(queue) {
+        try {
+            if (typeof queue.getWorkersCount === "function") {
+                return await queue.getWorkersCount();
+            }
+            if (typeof queue.getWorkers === "function") {
+                return (await queue.getWorkers()).length;
+            }
+        }
+        catch (error) {
+            logger.error("ConnectorPlatformHealth: worker count probe failed.");
+            logger.error(error);
+        }
+        return null;
+    }
+    /*
+     * Deployment-level checks. The same three for every connection.
+     */
+    static toPlatformChecks(status) {
+        const startedAtMs = Date.now();
+        const checks = [];
+        if (status.workerConsumers === null) {
+            checks.push(makeCheck({
+                key: "worker-consumers",
+                name: "Background workers",
+                status: "warn",
+                startedAtMs,
+                message: "Could not determine how many processes consume the Worker queue.",
+                remediation: "Check that Redis is reachable from the API and worker processes.",
+            }));
+        }
+        else if (status.workerConsumers === 0) {
+            checks.push(makeCheck({
+                key: "worker-consumers",
+                name: "Background workers",
+                status: "fail",
+                startedAtMs,
+                message: "No process is consuming the Worker queue, so scheduled polls, on-demand runs and imports never execute.",
+                remediation: "Set DISABLE_QUEUE_WORKERS=false on the app (the config.example.env default) or run the dedicated worker deployment (Helm: worker.enabled: true). A connection test from this page does not need a worker, which is why it can report this while polling stays silent.",
+                details: { workerConsumers: 0 },
+            }));
+        }
+        else {
+            checks.push(makeCheck({
+                key: "worker-consumers",
+                name: "Background workers",
+                status: "pass",
+                startedAtMs,
+                message: `${status.workerConsumers} process${status.workerConsumers === 1 ? "" : "es"} consuming the Worker queue.`,
+                details: Object.assign(Object.assign({ workerConsumers: status.workerConsumers }, (status.queueWaiting !== null
+                    ? { queueWaiting: status.queueWaiting }
+                    : {})), (status.queueFailed !== null
+                    ? { queueFailed: status.queueFailed }
+                    : {})),
+            }));
+        }
+        if (status.schedulerRegistered === null) {
+            checks.push(makeCheck({
+                key: "scheduler",
+                name: "Poll scheduler",
+                status: "warn",
+                startedAtMs,
+                message: "Could not read the scheduler registration from the queue.",
+                remediation: "Check that Redis is reachable and that the Workers feature set started without errors.",
+            }));
+        }
+        else if (!status.schedulerRegistered) {
+            checks.push(makeCheck({
+                key: "scheduler",
+                name: "Poll scheduler",
+                status: "fail",
+                startedAtMs,
+                message: "The minute-cadence poll scheduler is not registered, so no connection is ever enqueued for polling.",
+                remediation: "The Workers feature set registers the scheduler when a process boots. Restart the app or worker and check its startup logs for Workers errors.",
+            }));
+        }
+        else {
+            checks.push(makeCheck(Object.assign({ key: "scheduler", name: "Poll scheduler", status: "pass", startedAtMs, message: status.schedulerNextRunAt
+                    ? `Registered; next tick at ${status.schedulerNextRunAt}.`
+                    : "Registered." }, (status.schedulerNextRunAt
+                ? { details: { nextRunAt: status.schedulerNextRunAt } }
+                : {}))));
+        }
+        if (status.storageReachable === null) {
+            checks.push(makeCheck({
+                key: "storage",
+                name: "Security event storage",
+                status: "warn",
+                startedAtMs,
+                message: "Could not probe the analytics database.",
+            }));
+        }
+        else if (!status.storageReachable) {
+            checks.push(makeCheck({
+                key: "storage",
+                name: "Security event storage",
+                status: "fail",
+                startedAtMs,
+                message: "The analytics database that stores security events did not answer, so imported records cannot be written.",
+                remediation: "Check ClickHouse health and the CLICKHOUSE_* connection settings on the API and worker processes.",
+            }));
+        }
+        else {
+            checks.push(makeCheck({
+                key: "storage",
+                name: "Security event storage",
+                status: "pass",
+                startedAtMs,
+                message: "The analytics database answered.",
+            }));
+        }
+        return checks;
+    }
+    /*
+     * The connection's own schedule, read off its row. Only meaningful for a
+     * saved connection; the unsaved-settings test skips it.
+     */
+    static toScheduleCheck(facts, now = OneUptimeDate.getCurrentDate()) {
+        const startedAtMs = Date.now();
+        const intervalMs = Math.max(1, facts.pollIntervalInMinutes || 5) * 60 * 1000;
+        if (!facts.isEnabled) {
+            return makeCheck({
+                key: "connection-schedule",
+                name: "Scheduled polling",
+                status: "warn",
+                startedAtMs,
+                message: "Scheduled polling is disabled for this connection, so nothing is imported automatically.",
+                remediation: "Enable the connection to poll on its interval. On-demand runs and imports still work while it is disabled.",
+            });
+        }
+        if (facts.pendingRunCreatedAt &&
+            now.getTime() - facts.pendingRunCreatedAt.getTime() >
+                CONNECTOR_OVERDUE_GRACE_IN_MS) {
+            return makeCheck({
+                key: "connection-schedule",
+                name: "Scheduled polling",
+                status: "fail",
+                startedAtMs,
+                message: `A run has been queued since ${facts.pendingRunCreatedAt.toISOString()} and no worker has picked it up. New polls are blocked until it finishes or expires (20 minutes).`,
+                remediation: "Fix the background workers (see the Background workers check). The stale run expires on its own and polling resumes.",
+            });
+        }
+        if (!facts.lastPolledAt) {
+            const ageMs = facts.createdAt
+                ? now.getTime() - facts.createdAt.getTime()
+                : 0;
+            if (ageMs > CONNECTOR_OVERDUE_GRACE_IN_MS) {
+                return makeCheck({
+                    key: "connection-schedule",
+                    name: "Scheduled polling",
+                    status: "fail",
+                    startedAtMs,
+                    message: `This connection was created ${Math.round(ageMs / 60000)} minutes ago and has never been polled.`,
+                    remediation: "Polling is driven by the background workers and the poll scheduler; see those checks. If both pass, use Run now to trigger a poll and read its run history.",
+                });
+            }
+            return makeCheck({
+                key: "connection-schedule",
+                name: "Scheduled polling",
+                status: "pass",
+                startedAtMs,
+                message: "Not polled yet; the first scheduled poll runs within the next scheduler tick.",
+            });
+        }
+        const dueAtMs = facts.lastPolledAt.getTime() + intervalMs;
+        if (now.getTime() > dueAtMs + CONNECTOR_OVERDUE_GRACE_IN_MS) {
+            return makeCheck({
+                key: "connection-schedule",
+                name: "Scheduled polling",
+                status: "fail",
+                startedAtMs,
+                message: `The last poll attempt was at ${facts.lastPolledAt.toISOString()}, which is later than the ${facts.pollIntervalInMinutes} minute interval allows.`,
+                remediation: "Polls are overdue. Check the Background workers and Poll scheduler checks, then the worker logs.",
+                details: {
+                    lastPolledAt: facts.lastPolledAt.toISOString(),
+                    intervalMinutes: facts.pollIntervalInMinutes,
+                },
+            });
+        }
+        if (facts.lastError) {
+            return makeCheck({
+                key: "connection-schedule",
+                name: "Scheduled polling",
+                status: "fail",
+                startedAtMs,
+                message: `The last poll attempt failed: ${facts.lastError}`,
+                remediation: "Read the error prefix to see which step failed. The checks above exercise the same steps live.",
+                details: Object.assign({ lastPolledAt: facts.lastPolledAt.toISOString() }, (facts.lastSuccessfulPollAt
+                    ? { lastSuccessfulPollAt: facts.lastSuccessfulPollAt.toISOString() }
+                    : {})),
+            });
+        }
+        return makeCheck({
+            key: "connection-schedule",
+            name: "Scheduled polling",
+            status: "pass",
+            startedAtMs,
+            message: facts.lastEventIngestedAt
+                ? `Polling on schedule; last poll ${facts.lastPolledAt.toISOString()}, last event imported ${facts.lastEventIngestedAt.toISOString()}.`
+                : `Polling on schedule; last poll ${facts.lastPolledAt.toISOString()}. No event has been imported yet, which is expected while the source creates none.`,
+            details: Object.assign(Object.assign({ lastPolledAt: facts.lastPolledAt.toISOString() }, (facts.lastSuccessfulPollAt
+                ? { lastSuccessfulPollAt: facts.lastSuccessfulPollAt.toISOString() }
+                : {})), (facts.lastEventIngestedAt
+                ? { lastEventIngestedAt: facts.lastEventIngestedAt.toISOString() }
+                : {})),
+        });
+    }
+    /*
+     * Exported for tests that hand-build a queue: the real BullMQ class is
+     * only referenced for its type.
+     */
+    static isQueueLike(value) {
+        return Boolean(value) && typeof value === "object";
+    }
+}
+//# sourceMappingURL=ConnectorPlatformHealth.js.map

package/build/dist/Server/Utils/SecurityEvent/Connectors/ConnectorPlatformHealth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConnectorPlatformHealth.js","sourceRoot":"","sources":["../../../../../../Server/Utils/SecurityEvent/Connectors/ConnectorPlatformHealth.ts"],"names":[],"mappings":"AACA,OAAO,aAAa,MAAM,wBAAwB,CAAC;AAKnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,4CAA4C,CAAC;AACnF,OAAO,KAAK,EAAE,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEpC;;;;;;;;;;;;;GAaG;AAEH;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAkB;IAC1D,4CAA4C;IAC5C,6CAA6C;CAC9C,CAAC;AAEF,gEAAgE;AAChE,MAAM,CAAC,MAAM,6BAA6B,GAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AA6CnE,MAAM,CAAC,OAAO,OAAO,uBAAuB;IACnC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAGrC;QACC,MAAM,MAAM,GAA4B;YACtC,eAAe,EAAE,IAAI;YACrB,mBAAmB,EAAE,IAAI;YACzB,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,IAAI;YACjB,gBAAgB,EAAE,IAAI;SACvB,CAAC;QAEF,IAAI,KAAK,GAAqB,IAAI,CAAC;QAEnC,IAAI,CAAC;YACH,KAAK;gBACH,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,aAAa;oBAClB,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAA0B,CAAC;QAC/D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,gEAAgE,CACjE,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,eAAe,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAE5D,IAAI,CAAC;gBACH,IAAI,OAAO,KAAK,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;oBACjD,MAAM,UAAU,GACd,MAAM,KAAK,CAAC,gBAAgB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;oBAC9C,MAAM,OAAO,GAAmC,EAAE,CAAC;oBAEnD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;wBACnC,IAAI,SAAS,IAAI,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,EAAE,CAAC;4BACtD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;wBAC1B,CAAC;oBACH,CAAC;oBAED,MAAM,CAAC,mBAAmB,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;oBAChD,MAAM,QAAQ,GAAkB,OAAO;yBACpC,GAAG,CAAC,CAAC,SAAkC,EAAU,EAAE;wBAClD,OAAO,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ;4BACvC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;4BAC/B,CAAC,CAAC,SAAS,CAAC,IAAI;4BAChB,CAAC,CAAC,CAAC,CAAC;oBACR,CAAC,CAAC;yBACD,MAAM,CAAC,CAAC,KAAa,EAAW,EAAE;wBACjC,OAAO,KAAK,GAAG,CAAC,CAAC;oBACnB,CAAC,CAAC,CAAC;oBACL,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACxB,MAAM,CAAC,kBAAkB,GAAG,IAAI,IAAI,CAClC,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,CACtB,CAAC,WAAW,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;gBACjE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,CAAC,YAAY,GAAG,MAAM,KAAK,CAAC,eAAe,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;gBACrE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,CAAC,WAAW,GAAG,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC;YACpD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;gBACpE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,CAAC,gBAAgB,GAAG,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,oBAAoB;gBAClD,CAAC,CAAC,MAAM,IAAI,CAAC,oBAAoB,EAAE;gBACnC,CAAC,CAAC,MAAM,qBAAqB,CAAC,sBAAsB,EAAE,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAC/D,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAClC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;;;OASG;IACI,MAAM,CAAC,oBAAoB,CAChC,SAAkC;QAElC,MAAM,IAAI,GACR,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,MAAM,GAAG,GAAW,OAAO,SAAS,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAE3E,OAAO,6BAA6B,CAAC,IAAI,CAAC,CAAC,OAAe,EAAW,EAAE;YACrE,OAAO,IAAI,KAAK,OAAO,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,gBAAgB,CACnC,KAAgB;QAEhB,IAAI,CAAC;YACH,IAAI,OAAO,KAAK,CAAC,eAAe,KAAK,UAAU,EAAE,CAAC;gBAChD,OAAO,MAAM,KAAK,CAAC,eAAe,EAAE,CAAC;YACvC,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBAC3C,OAAO,CAAC,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC;YAC3C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACpE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,gBAAgB,CAC5B,MAA+B;QAE/B,MAAM,WAAW,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;QACvC,MAAM,MAAM,GAAkC,EAAE,CAAC;QAEjD,IAAI,MAAM,CAAC,eAAe,KAAK,IAAI,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CACT,SAAS,CAAC;gBACR,GAAG,EAAE,kBAAkB;gBACvB,IAAI,EAAE,oBAAoB;gBAC1B,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EACL,kEAAkE;gBACpE,WAAW,EACT,kEAAkE;aACrE,CAAC,CACH,CAAC;QACJ,CAAC;aAAM,IAAI,MAAM,CAAC,eAAe,KAAK,CAAC,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CACT,SAAS,CAAC;gBACR,GAAG,EAAE,kBAAkB;gBACvB,IAAI,EAAE,oBAAoB;gBAC1B,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EACL,yGAAyG;gBAC3G,WAAW,EACT,uQAAuQ;gBACzQ,OAAO,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE;aAChC,CAAC,CACH,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CACT,SAAS,CAAC;gBACR,GAAG,EAAE,kBAAkB;gBACvB,IAAI,EAAE,oBAAoB;gBAC1B,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EAAE,GAAG,MAAM,CAAC,eAAe,WAAW,MAAM,CAAC,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,8BAA8B;gBACnH,OAAO,gCACL,eAAe,EAAE,MAAM,CAAC,eAAe,IACpC,CAAC,MAAM,CAAC,YAAY,KAAK,IAAI;oBAC9B,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE;oBACvC,CAAC,CAAC,EAAE,CAAC,GACJ,CAAC,MAAM,CAAC,WAAW,KAAK,IAAI;oBAC7B,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE;oBACrC,CAAC,CAAC,EAAE,CAAC,CACR;aACF,CAAC,CACH,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,mBAAmB,KAAK,IAAI,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CACT,SAAS,CAAC;gBACR,GAAG,EAAE,WAAW;gBAChB,IAAI,EAAE,gBAAgB;gBACtB,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EAAE,2DAA2D;gBACpE,WAAW,EACT,wFAAwF;aAC3F,CAAC,CACH,CAAC;QACJ,CAAC;aAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CACT,SAAS,CAAC;gBACR,GAAG,EAAE,WAAW;gBAChB,IAAI,EAAE,gBAAgB;gBACtB,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EACL,qGAAqG;gBACvG,WAAW,EACT,gJAAgJ;aACnJ,CAAC,CACH,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CACT,SAAS,iBACP,GAAG,EAAE,WAAW,EAChB,IAAI,EAAE,gBAAgB,EACtB,MAAM,EAAE,MAAM,EACd,WAAW,EACX,OAAO,EAAE,MAAM,CAAC,kBAAkB;oBAChC,CAAC,CAAC,4BAA4B,MAAM,CAAC,kBAAkB,GAAG;oBAC1D,CAAC,CAAC,aAAa,IACd,CAAC,MAAM,CAAC,kBAAkB;gBAC3B,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,kBAAkB,EAAE,EAAE;gBACvD,CAAC,CAAC,EAAE,CAAC,EACP,CACH,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CACT,SAAS,CAAC;gBACR,GAAG,EAAE,SAAS;gBACd,IAAI,EAAE,wBAAwB;gBAC9B,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EAAE,yCAAyC;aACnD,CAAC,CACH,CAAC;QACJ,CAAC;aAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CACT,SAAS,CAAC;gBACR,GAAG,EAAE,SAAS;gBACd,IAAI,EAAE,wBAAwB;gBAC9B,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EACL,2GAA2G;gBAC7G,WAAW,EACT,mGAAmG;aACtG,CAAC,CACH,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CACT,SAAS,CAAC;gBACR,GAAG,EAAE,SAAS;gBACd,IAAI,EAAE,wBAAwB;gBAC9B,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EAAE,kCAAkC;aAC5C,CAAC,CACH,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,eAAe,CAC3B,KAA6B,EAC7B,MAAY,aAAa,CAAC,cAAc,EAAE;QAE1C,MAAM,WAAW,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;QACvC,MAAM,UAAU,GACd,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,qBAAqB,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;QAE5D,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,SAAS,CAAC;gBACf,GAAG,EAAE,qBAAqB;gBAC1B,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EACL,0FAA0F;gBAC5F,WAAW,EACT,4GAA4G;aAC/G,CAAC,CAAC;QACL,CAAC;QAED,IACE,KAAK,CAAC,mBAAmB;YACzB,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,mBAAmB,CAAC,OAAO,EAAE;gBACjD,6BAA6B,EAC/B,CAAC;YACD,OAAO,SAAS,CAAC;gBACf,GAAG,EAAE,qBAAqB;gBAC1B,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EAAE,+BAA+B,KAAK,CAAC,mBAAmB,CAAC,WAAW,EAAE,mGAAmG;gBAClL,WAAW,EACT,sHAAsH;aACzH,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YACxB,MAAM,KAAK,GAAW,KAAK,CAAC,SAAS;gBACnC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE;gBAC3C,CAAC,CAAC,CAAC,CAAC;YAEN,IAAI,KAAK,GAAG,6BAA6B,EAAE,CAAC;gBAC1C,OAAO,SAAS,CAAC;oBACf,GAAG,EAAE,qBAAqB;oBAC1B,IAAI,EAAE,mBAAmB;oBACzB,MAAM,EAAE,MAAM;oBACd,WAAW;oBACX,OAAO,EAAE,+BAA+B,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,yCAAyC;oBAC1G,WAAW,EACT,6JAA6J;iBAChK,CAAC,CAAC;YACL,CAAC;YAED,OAAO,SAAS,CAAC;gBACf,GAAG,EAAE,qBAAqB;gBAC1B,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EACL,+EAA+E;aAClF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAW,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,UAAU,CAAC;QAElE,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,GAAG,6BAA6B,EAAE,CAAC;YAC5D,OAAO,SAAS,CAAC;gBACf,GAAG,EAAE,qBAAqB;gBAC1B,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EAAE,gCAAgC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,6BAA6B,KAAK,CAAC,qBAAqB,0BAA0B;gBAC3J,WAAW,EACT,kGAAkG;gBACpG,OAAO,EAAE;oBACP,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE;oBAC9C,eAAe,EAAE,KAAK,CAAC,qBAAqB;iBAC7C;aACF,CAAC,CAAC;QACL,CAAC;QAED,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO,SAAS,CAAC;gBACf,GAAG,EAAE,qBAAqB;gBAC1B,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,MAAM;gBACd,WAAW;gBACX,OAAO,EAAE,iCAAiC,KAAK,CAAC,SAAS,EAAE;gBAC3D,WAAW,EACT,gGAAgG;gBAClG,OAAO,kBACL,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,IAC3C,CAAC,KAAK,CAAC,oBAAoB;oBAC5B,CAAC,CAAC,EAAE,oBAAoB,EAAE,KAAK,CAAC,oBAAoB,CAAC,WAAW,EAAE,EAAE;oBACpE,CAAC,CAAC,EAAE,CAAC,CACR;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;YACf,GAAG,EAAE,qBAAqB;YAC1B,IAAI,EAAE,mBAAmB;YACzB,MAAM,EAAE,MAAM;YACd,WAAW;YACX,OAAO,EAAE,KAAK,CAAC,mBAAmB;gBAChC,CAAC,CAAC,kCAAkC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,yBAAyB,KAAK,CAAC,mBAAmB,CAAC,WAAW,EAAE,GAAG;gBACvI,CAAC,CAAC,kCAAkC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,oFAAoF;YAC1J,OAAO,gCACL,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,IAC3C,CAAC,KAAK,CAAC,oBAAoB;gBAC5B,CAAC,CAAC,EAAE,oBAAoB,EAAE,KAAK,CAAC,oBAAoB,CAAC,WAAW,EAAE,EAAE;gBACpE,CAAC,CAAC,EAAE,CAAC,GACJ,CAAC,KAAK,CAAC,mBAAmB;gBAC3B,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,CAAC,WAAW,EAAE,EAAE;gBAClE,CAAC,CAAC,EAAE,CAAC,CACR;SACF,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,WAAW,CAAC,KAAc;QACtC,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC;IACrD,CAAC;CACF"}