@oneuptime/common 12.0.25 → 12.0.26

package/build/dist/Server/Utils/NetworkDevice/DeviceHealthAggregation.js

@@ -0,0 +1,332 @@
+import SortOrder from "../../../Types/BaseDatabase/SortOrder";
+import AggregateResultUtil from "../../Types/Database/AggregateResultUtil";
+import { DEFAULT_DEVICE_POLLING_INTERVAL_IN_MINUTES, DEVICE_MIN_STALE_WINDOW_IN_MINUTES, DEVICE_MISSED_POLL_ALLOWANCE, } from "../../../Utils/NetworkDevice/DeviceReachabilityUtil";
+import OneUptimeDate from "../../../Types/Date";
+/*
+ * Counting device health for a whole estate WITHOUT loading the estate.
+ *
+ * The obvious way to make a per-site health rollup fast is to write the
+ * classification rule a second time, in SQL: `CASE WHEN isReachable = false
+ * THEN 'down' WHEN ...`. That is also the way to guarantee the product
+ * eventually contradicts itself — a device the site card counts as healthy and
+ * the topology map next to it draws red — which is the exact outcome
+ * DeviceHealthStateUtil was created to make impossible. A rule with two
+ * implementations has two behaviours the moment either one is edited.
+ *
+ * So the rule does not move. What moves is the DATA REDUCTION.
+ *
+ * `deviceHealthState` is a pure function of six facts about a row, and five of
+ * them are already booleans or three-state flags. Group the table by those
+ * facts in SQL and 80,000 rows collapse into at most a few per site — one
+ * bucket per distinct combination that actually occurs. Then hand each bucket
+ * back to the REAL classifier once and multiply its verdict by the bucket's
+ * count.
+ *
+ * One rule, one implementation, and the wire carries buckets instead of rows.
+ *
+ * The one fact that is not already discrete is staleness — "nothing has polled
+ * this device in a long time" — because the window is per-device, derived from
+ * each row's own interval. That predicate is computed in SQL below, from the
+ * very constants DeviceReachabilityUtil derives its window from, so there is
+ * still only one definition of how long "a long time" is.
+ */
+/**
+ * Column names on `NetworkDevice`, qualified with the alias
+ * `DatabaseService.aggregateBy` runs under (which is the model name).
+ */
+function column(columnName) {
+    return `"NetworkDevice"."${columnName}"`;
+}
+/*
+ * The staleness window, in minutes, as SQL — the exact expression
+ * `DeviceReachabilityUtil.getStaleWindowInMinutes` computes in TypeScript,
+ * built from the same three exported constants so the two cannot drift.
+ */
+const STALE_WINDOW_IN_MINUTES_SQL = `GREATEST(
+  (CASE
+    WHEN ${column("pollingIntervalInMinutes")} IS NULL
+      OR ${column("pollingIntervalInMinutes")} <= 0
+    THEN ${DEFAULT_DEVICE_POLLING_INTERVAL_IN_MINUTES}
+    ELSE GREATEST(${column("pollingIntervalInMinutes")}, 1)
+  END)::numeric * ${DEVICE_MISSED_POLL_ALLOWANCE},
+  ${DEVICE_MIN_STALE_WINDOW_IN_MINUTES}
+)`;
+/*
+ * Newest of the two contact timestamps — `DeviceReachabilityUtil`'s `newerOf`.
+ * Postgres GREATEST ignores NULL arguments and is NULL only when every
+ * argument is, which is precisely that function's behaviour.
+ */
+const LAST_CONTACT_AT_SQL = `GREATEST(${column("lastPolledAt")}, ${column("lastSeenAt")})`;
+/**
+ * The name of the bound parameter carrying "now".
+ *
+ * Passed in rather than using the database's `NOW()` so that a caller
+ * classifying several groups — or comparing an aggregate against a row it also
+ * fetched — measures every one of them against a single instant. `NOW()` would
+ * also be the transaction's start time rather than the caller's, which is a
+ * different clock from the one the TypeScript rule reads.
+ */
+export const DEVICE_HEALTH_NOW_PARAMETER = "deviceHealthNow";
+/** Aliases the grouped columns come back under. */
+export var DeviceHealthGroupAlias;
+(function (DeviceHealthGroupAlias) {
+    DeviceHealthGroupAlias["SiteId"] = "siteId";
+    DeviceHealthGroupAlias["MonitorStatusId"] = "monitorStatusId";
+    DeviceHealthGroupAlias["MonitoringMethod"] = "monitoringMethod";
+    DeviceHealthGroupAlias["IsReachable"] = "isReachable";
+    DeviceHealthGroupAlias["HasBeenPolled"] = "hasBeenPolled";
+    DeviceHealthGroupAlias["HasBeenSeen"] = "hasBeenSeen";
+    DeviceHealthGroupAlias["IsStale"] = "isStale";
+    DeviceHealthGroupAlias["HasDownInterfaces"] = "hasDownInterfaces";
+    DeviceHealthGroupAlias["DeviceCount"] = "deviceCount";
+    DeviceHealthGroupAlias["InterfacesDownTotal"] = "interfacesDownTotal";
+})(DeviceHealthGroupAlias || (DeviceHealthGroupAlias = {}));
+/*
+ * Every fact `deviceHealthState` reads, and nothing else.
+ *
+ * Reduced to its coarsest form on purpose: `interfacesDown` is grouped as
+ * "> 0 or not" rather than by value, and the timestamps as "set or not" plus
+ * the staleness verdict, because that is all the rule can distinguish. Grouping
+ * by the raw values instead would produce almost one bucket per device and
+ * defeat the whole exercise.
+ */
+const DEVICE_HEALTH_DISCRIMINATOR_COLUMNS = [
+    {
+        expression: column("currentMonitorStatusId"),
+        alias: DeviceHealthGroupAlias.MonitorStatusId,
+    },
+    /*
+     * Grouped raw rather than pre-parsed into "is monitor backed", because
+     * `NetworkDeviceMonitoringMethodUtil.parse` owns what an unrecognised or
+     * NULL value means and there is no reason for SQL to hold a second opinion.
+     * The column only ever holds a handful of distinct values, so the raw form
+     * costs nothing in bucket count.
+     *
+     * Note carefully WHO reads it: `deviceHealthState` never receives it (it is
+     * not part of `DeviceHealthStateInput`, and `deviceHealthInputForGroup` does
+     * not emit it), so the per-site health rollups are indifferent to it. Its
+     * one consumer is the Overview's fleet tally, which re-attaches it and calls
+     * `DeviceReachabilityUtil` directly — and a tally that dropped it would
+     * report every ping-only device as Pending forever (issue #3392, one level
+     * up).
+     *
+     * It therefore costs the rollup path a discriminator it does not use. On a
+     * real fleet that costs nothing, because `monitoringMethod` is almost
+     * perfectly correlated with whether the device carries a stamped monitor
+     * status — which is already a group key — so it splits no bucket that
+     * `currentMonitorStatusId` had not split anyway. Keeping one discriminator
+     * set is worth more than saving that.
+     */
+    {
+        expression: column("monitoringMethod"),
+        alias: DeviceHealthGroupAlias.MonitoringMethod,
+    },
+    {
+        expression: column("isReachable"),
+        alias: DeviceHealthGroupAlias.IsReachable,
+    },
+    {
+        expression: `(${column("lastPolledAt")} IS NOT NULL)`,
+        alias: DeviceHealthGroupAlias.HasBeenPolled,
+    },
+    {
+        expression: `(${column("lastSeenAt")} IS NOT NULL)`,
+        alias: DeviceHealthGroupAlias.HasBeenSeen,
+    },
+    /*
+     * Guarded by the only branch that can read it.
+     *
+     * `DeviceReachabilityUtil` lets staleness decide up-vs-down in exactly one
+     * case — a row with no recorded poll outcome that has nevertheless been seen
+     * (`isReachable IS NULL AND lastSeenAt IS NOT NULL`), which is a row written
+     * before the outcome column existed. Everywhere else staleness is an
+     * annotation the health rule ignores entirely.
+     *
+     * So the interval arithmetic is skipped for every row that cannot be
+     * affected by it — which, after the backfill migration, is essentially all
+     * of them. The guard is not an optimisation that changes the answer: for any
+     * row outside that branch the classifier's verdict is identical whether this
+     * says true or false, and constant-false keeps the buckets from splitting on
+     * a fact nothing reads.
+     */
+    {
+        /*
+         * Written as "how many minutes since last contact, compared against the
+         * window" rather than as "before now minus an interval", which is both
+         * the shape `DeviceReachabilityUtil` uses and the one that cannot
+         * overflow.
+         *
+         * The interval form — `now - make_interval(mins => window)` — takes an
+         * `int` parameter, and `pollingIntervalInMinutes` is an unbounded `integer`
+         * column with no CHECK behind it. Any value above 214,748,364 overflows
+         * `int4` on the `* 10`, and Postgres aborts the STATEMENT: one absurd row
+         * would 500 the overview, the hierarchy drill-down and every rollup for
+         * the whole project. The TypeScript twin is a double and has no such
+         * ceiling, so that was a genuine divergence rather than a shared limit.
+         * `::numeric` and a subtraction have neither.
+         */
+        expression: `(
+      CASE WHEN ${column("isReachable")} IS NULL
+                AND ${column("lastSeenAt")} IS NOT NULL
+      THEN (
+        ${LAST_CONTACT_AT_SQL} IS NOT NULL
+        AND EXTRACT(
+          EPOCH FROM (:${DEVICE_HEALTH_NOW_PARAMETER}::timestamptz - ${LAST_CONTACT_AT_SQL})
+        ) / 60 > ${STALE_WINDOW_IN_MINUTES_SQL}
+      )
+      ELSE false END
+    )`,
+        alias: DeviceHealthGroupAlias.IsStale,
+    },
+    {
+        expression: `(COALESCE(${column("interfacesDown")}, 0) > 0)`,
+        alias: DeviceHealthGroupAlias.HasDownInterfaces,
+    },
+];
+/** How many devices are in a bucket. */
+export const DEVICE_COUNT_AGGREGATE = {
+    expression: "COUNT(*)",
+    alias: DeviceHealthGroupAlias.DeviceCount,
+};
+/**
+ * How many INTERFACES are down across a bucket's devices.
+ *
+ * A real SUM, not a count of devices with dark ports: a switch with three
+ * down interfaces contributes three. Carried alongside the bucket counts so a
+ * caller that wants both the fleet's health tally and its total dark ports
+ * gets them from one statement.
+ */
+export const INTERFACES_DOWN_AGGREGATE = {
+    expression: `COALESCE(SUM(${column("interfacesDown")}), 0)`,
+    alias: DeviceHealthGroupAlias.InterfacesDownTotal,
+};
+/** Both aggregates, which is what every health grouping selects. */
+export const DEVICE_HEALTH_AGGREGATES = [
+    DEVICE_COUNT_AGGREGATE,
+    INTERFACES_DOWN_AGGREGATE,
+];
+/** Health buckets for the whole matched set, with no per-site breakdown. */
+export const DEVICE_HEALTH_GROUP_COLUMNS = [
+    ...DEVICE_HEALTH_DISCRIMINATOR_COLUMNS,
+];
+/** Health buckets broken down by the site each device is attached to. */
+export const DEVICE_HEALTH_GROUP_COLUMNS_BY_SITE = [
+    { expression: column("siteId"), alias: DeviceHealthGroupAlias.SiteId },
+    ...DEVICE_HEALTH_DISCRIMINATOR_COLUMNS,
+];
+/**
+ * A stable order for the buckets.
+ *
+ * `SiteStatusRollupUtil.worstStatus` documents that a priority tie keeps the
+ * FIRST contributor. Buckets arrive in whatever order a HashAggregate produces
+ * them, which is not merely unspecified but can differ between two executions
+ * of the same statement — so without this, a project that ever defined two
+ * MonitorStatus rows at the same priority could see a site's rolled-up status
+ * flip between page loads with nothing having changed.
+ *
+ * Sorting a few thousand already-aggregated rows costs nothing next to the
+ * scan that produced them.
+ */
+export const DEVICE_HEALTH_GROUP_ORDER = [
+    {
+        expression: `"${DeviceHealthGroupAlias.MonitorStatusId}"`,
+        sortOrder: SortOrder.Ascending,
+    },
+    {
+        expression: `"${DeviceHealthGroupAlias.DeviceCount}"`,
+        sortOrder: SortOrder.Descending,
+    },
+];
+export function parseDeviceHealthGroup(row) {
+    return {
+        siteId: AggregateResultUtil.toStringOrNull(row, DeviceHealthGroupAlias.SiteId),
+        monitorStatusId: AggregateResultUtil.toStringOrNull(row, DeviceHealthGroupAlias.MonitorStatusId),
+        monitoringMethod: AggregateResultUtil.toStringOrNull(row, DeviceHealthGroupAlias.MonitoringMethod),
+        isReachable: AggregateResultUtil.toNullableBoolean(row, DeviceHealthGroupAlias.IsReachable),
+        hasBeenPolled: AggregateResultUtil.toBoolean(row, DeviceHealthGroupAlias.HasBeenPolled),
+        hasBeenSeen: AggregateResultUtil.toBoolean(row, DeviceHealthGroupAlias.HasBeenSeen),
+        isStale: AggregateResultUtil.toBoolean(row, DeviceHealthGroupAlias.IsStale),
+        hasDownInterfaces: AggregateResultUtil.toBoolean(row, DeviceHealthGroupAlias.HasDownInterfaces),
+        deviceCount: AggregateResultUtil.toNumber(row, DeviceHealthGroupAlias.DeviceCount),
+        interfacesDownTotal: AggregateResultUtil.toNumber(row, DeviceHealthGroupAlias.InterfacesDownTotal),
+    };
+}
+/*
+ * How far back a synthetic "stale" timestamp is placed.
+ *
+ * `deviceHealthInputForGroup` hands the classifier dates rather than the
+ * staleness flag itself, because the flag is not part of the classifier's
+ * input shape — it is something the classifier derives. So the dates have to
+ * be chosen to make it derive the answer SQL already reached: comfortably
+ * outside any window `getStaleWindowInMinutes` can return for the default
+ * interval (60 minutes) to mean stale, and the caller's own "now" to mean
+ * fresh.
+ */
+const SYNTHETIC_STALE_AGE_IN_MINUTES = 365 * 24 * 60;
+/**
+ * One bucket, expressed as the input the REAL classifier takes.
+ *
+ * The dates are synthetic and deliberately so: SQL has already decided
+ * staleness per device, against each device's own interval, and a bucket has
+ * no single interval to re-derive it from. Feeding back the default interval
+ * plus a timestamp on the correct side of that window reproduces the verdict
+ * exactly — and, importantly, leaves the decision itself with
+ * `deviceHealthState` rather than copying it here.
+ */
+export function deviceHealthInputForGroup(data) {
+    const { group, now } = data;
+    const contactAt = group.isStale
+        ? OneUptimeDate.addRemoveMinutes(now, -SYNTHETIC_STALE_AGE_IN_MINUTES)
+        : now;
+    return {
+        monitorStatusIsOffline: data.monitorStatusIsOffline,
+        isReachable: group.isReachable,
+        lastPolledAt: group.hasBeenPolled ? contactAt : null,
+        lastSeenAt: group.hasBeenSeen ? contactAt : null,
+        /*
+         * Left unset on purpose. The window has already been applied in SQL
+         * against each device's real interval; re-supplying an interval here
+         * would ask the classifier to re-derive staleness from a bucket that no
+         * longer has one. Unset reads as the column default, which is the window
+         * the synthetic dates above are placed relative to.
+         */
+        pollingIntervalInMinutes: undefined,
+        interfacesDown: group.hasDownInterfaces ? 1 : 0,
+    };
+}
+/**
+ * One bucket, expressed as the input the site ROLLUP rule takes.
+ *
+ * `SiteStatusRollupUtil` answers a different question from
+ * `deviceHealthState` — which MonitorStatus a site inherits, rather than how
+ * healthy one device is — but it reads the same reachability facts, so the
+ * same buckets serve it.
+ *
+ * `deviceCount` is carried through because the two rollup policies disagree
+ * about whether it matters. Worst-of is indifferent: one representative per
+ * bucket produces the identical winner that every device in the bucket
+ * would have. A percentage is not — a bucket of four hundred healthy devices
+ * and a bucket of one dark device are the same NUMBER of buckets, and
+ * scoring the share off buckets would read that region as 50% down.
+ *
+ * The synthetic timestamps are chosen exactly as in
+ * `deviceHealthInputForGroup`, and for the same reason.
+ */
+export function deviceRollupStateForGroup(data) {
+    const { group, now } = data;
+    const contactAt = group.isStale
+        ? OneUptimeDate.addRemoveMinutes(now, -SYNTHETIC_STALE_AGE_IN_MINUTES)
+        : now;
+    return {
+        currentMonitorStatusId: group.monitorStatusId,
+        monitorStatusPriority: data.monitorStatusPriority,
+        monitorStatusIsOperational: data.monitorStatusIsOperational,
+        isReachable: group.isReachable,
+        lastPolledAt: group.hasBeenPolled ? contactAt : null,
+        lastSeenAt: group.hasBeenSeen ? contactAt : null,
+        // See deviceHealthInputForGroup — staleness is already applied.
+        pollingIntervalInMinutes: undefined,
+        deviceCount: group.deviceCount,
+    };
+}
+//# sourceMappingURL=DeviceHealthAggregation.js.map

package/build/dist/Server/Utils/NetworkDevice/DeviceHealthAggregation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DeviceHealthAggregation.js","sourceRoot":"","sources":["../../../../../Server/Utils/NetworkDevice/DeviceHealthAggregation.ts"],"names":[],"mappings":"AAKA,OAAO,SAAS,MAAM,uCAAuC,CAAC;AAC9D,OAAO,mBAAmB,MAAM,0CAA0C,CAAC;AAE3E,OAAO,EACL,0CAA0C,EAC1C,kCAAkC,EAClC,4BAA4B,GAC7B,MAAM,qDAAqD,CAAC;AAC7D,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAGhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH;;;GAGG;AACH,SAAS,MAAM,CAAC,UAAkB;IAChC,OAAO,oBAAoB,UAAU,GAAG,CAAC;AAC3C,CAAC;AAED;;;;GAIG;AACH,MAAM,2BAA2B,GAAW;;WAEjC,MAAM,CAAC,0BAA0B,CAAC;WAClC,MAAM,CAAC,0BAA0B,CAAC;WAClC,0CAA0C;oBACjC,MAAM,CAAC,0BAA0B,CAAC;oBAClC,4BAA4B;IAC5C,kCAAkC;EACpC,CAAC;AAEH;;;;GAIG;AACH,MAAM,mBAAmB,GAAW,YAAY,MAAM,CACpD,cAAc,CACf,KAAK,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC;AAE9B;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAW,iBAAiB,CAAC;AAErE,mDAAmD;AACnD,MAAM,CAAN,IAAY,sBAWX;AAXD,WAAY,sBAAsB;IAChC,2CAAiB,CAAA;IACjB,6DAAmC,CAAA;IACnC,+DAAqC,CAAA;IACrC,qDAA2B,CAAA;IAC3B,yDAA+B,CAAA;IAC/B,qDAA2B,CAAA;IAC3B,6CAAmB,CAAA;IACnB,iEAAuC,CAAA;IACvC,qDAA2B,CAAA;IAC3B,qEAA2C,CAAA;AAC7C,CAAC,EAXW,sBAAsB,KAAtB,sBAAsB,QAWjC;AAED;;;;;;;;GAQG;AACH,MAAM,mCAAmC,GAA2B;IAClE;QACE,UAAU,EAAE,MAAM,CAAC,wBAAwB,CAAC;QAC5C,KAAK,EAAE,sBAAsB,CAAC,eAAe;KAC9C;IACD;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH;QACE,UAAU,EAAE,MAAM,CAAC,kBAAkB,CAAC;QACtC,KAAK,EAAE,sBAAsB,CAAC,gBAAgB;KAC/C;IACD;QACE,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC;QACjC,KAAK,EAAE,sBAAsB,CAAC,WAAW;KAC1C;IACD;QACE,UAAU,EAAE,IAAI,MAAM,CAAC,cAAc,CAAC,eAAe;QACrD,KAAK,EAAE,sBAAsB,CAAC,aAAa;KAC5C;IACD;QACE,UAAU,EAAE,IAAI,MAAM,CAAC,YAAY,CAAC,eAAe;QACnD,KAAK,EAAE,sBAAsB,CAAC,WAAW;KAC1C;IACD;;;;;;;;;;;;;;;OAeG;IACH;QACE;;;;;;;;;;;;;;WAcG;QACH,UAAU,EAAE;kBACE,MAAM,CAAC,aAAa,CAAC;sBACjB,MAAM,CAAC,YAAY,CAAC;;UAEhC,mBAAmB;;yBAEJ,2BAA2B,mBAAmB,mBAAmB;mBACvE,2BAA2B;;;MAGxC;QACF,KAAK,EAAE,sBAAsB,CAAC,OAAO;KACtC;IACD;QACE,UAAU,EAAE,aAAa,MAAM,CAAC,gBAAgB,CAAC,WAAW;QAC5D,KAAK,EAAE,sBAAsB,CAAC,iBAAiB;KAChD;CACF,CAAC;AAEF,wCAAwC;AACxC,MAAM,CAAC,MAAM,sBAAsB,GAAoB;IACrD,UAAU,EAAE,UAAU;IACtB,KAAK,EAAE,sBAAsB,CAAC,WAAW;CAC1C,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAoB;IACxD,UAAU,EAAE,gBAAgB,MAAM,CAAC,gBAAgB,CAAC,OAAO;IAC3D,KAAK,EAAE,sBAAsB,CAAC,mBAAmB;CAClD,CAAC;AAEF,oEAAoE;AACpE,MAAM,CAAC,MAAM,wBAAwB,GAA2B;IAC9D,sBAAsB;IACtB,yBAAyB;CAC1B,CAAC;AAEF,4EAA4E;AAC5E,MAAM,CAAC,MAAM,2BAA2B,GAA2B;IACjE,GAAG,mCAAmC;CACvC,CAAC;AAEF,yEAAyE;AACzE,MAAM,CAAC,MAAM,mCAAmC,GAA2B;IACzE,EAAE,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,sBAAsB,CAAC,MAAM,EAAE;IACtE,GAAG,mCAAmC;CACvC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAA0B;IAC9D;QACE,UAAU,EAAE,IAAI,sBAAsB,CAAC,eAAe,GAAG;QACzD,SAAS,EAAE,SAAS,CAAC,SAAS;KAC/B;IACD;QACE,UAAU,EAAE,IAAI,sBAAsB,CAAC,WAAW,GAAG;QACrD,SAAS,EAAE,SAAS,CAAC,UAAU;KAChC;CACF,CAAC;AAsBF,MAAM,UAAU,sBAAsB,CAAC,GAAiB;IACtD,OAAO;QACL,MAAM,EAAE,mBAAmB,CAAC,cAAc,CACxC,GAAG,EACH,sBAAsB,CAAC,MAAM,CAC9B;QACD,eAAe,EAAE,mBAAmB,CAAC,cAAc,CACjD,GAAG,EACH,sBAAsB,CAAC,eAAe,CACvC;QACD,gBAAgB,EAAE,mBAAmB,CAAC,cAAc,CAClD,GAAG,EACH,sBAAsB,CAAC,gBAAgB,CACxC;QACD,WAAW,EAAE,mBAAmB,CAAC,iBAAiB,CAChD,GAAG,EACH,sBAAsB,CAAC,WAAW,CACnC;QACD,aAAa,EAAE,mBAAmB,CAAC,SAAS,CAC1C,GAAG,EACH,sBAAsB,CAAC,aAAa,CACrC;QACD,WAAW,EAAE,mBAAmB,CAAC,SAAS,CACxC,GAAG,EACH,sBAAsB,CAAC,WAAW,CACnC;QACD,OAAO,EAAE,mBAAmB,CAAC,SAAS,CAAC,GAAG,EAAE,sBAAsB,CAAC,OAAO,CAAC;QAC3E,iBAAiB,EAAE,mBAAmB,CAAC,SAAS,CAC9C,GAAG,EACH,sBAAsB,CAAC,iBAAiB,CACzC;QACD,WAAW,EAAE,mBAAmB,CAAC,QAAQ,CACvC,GAAG,EACH,sBAAsB,CAAC,WAAW,CACnC;QACD,mBAAmB,EAAE,mBAAmB,CAAC,QAAQ,CAC/C,GAAG,EACH,sBAAsB,CAAC,mBAAmB,CAC3C;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,8BAA8B,GAAW,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC;AAE7D;;;;;;;;;GASG;AACH,MAAM,UAAU,yBAAyB,CAAC,IAKzC;IACC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAE5B,MAAM,SAAS,GAAS,KAAK,CAAC,OAAO;QACnC,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC,8BAA8B,CAAC;QACtE,CAAC,CAAC,GAAG,CAAC;IAER,OAAO;QACL,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;QACnD,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,YAAY,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACpD,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QAChD;;;;;;WAMG;QACH,wBAAwB,EAAE,SAAS;QACnC,cAAc,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAChD,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,yBAAyB,CAAC,IAWzC;IACC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAE5B,MAAM,SAAS,GAAS,KAAK,CAAC,OAAO;QACnC,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC,8BAA8B,CAAC;QACtE,CAAC,CAAC,GAAG,CAAC;IAER,OAAO;QACL,sBAAsB,EAAE,KAAK,CAAC,eAAe;QAC7C,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;QACjD,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;QAC3D,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,YAAY,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACpD,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QAChD,gEAAgE;QAChE,wBAAwB,EAAE,SAAS;QACnC,WAAW,EAAE,KAAK,CAAC,WAAW;KAC/B,CAAC;AACJ,CAAC"}

package/build/dist/Server/Utils/NetworkSite/NetworkSiteMaintenanceSuppression.js

@@ -0,0 +1,279 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import NetworkSiteService from "../../Services/NetworkSiteService";
+import ScheduledMaintenanceService from "../../Services/ScheduledMaintenanceService";
+import { LIMIT_PER_PROJECT } from "../../../Types/Database/LimitMax";
+import ObjectID from "../../../Types/ObjectID";
+import OneUptimeDate from "../../../Types/Date";
+import QueryHelper from "../../Types/Database/QueryHelper";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+import logger from "../Logger";
+/*
+ * Which NetworkSites a scheduled maintenance event is currently silencing,
+ * and which windows to subtract from a site's uptime.
+ *
+ * Network sites are the one maintenance-attachable resource whose health is
+ * COMPUTED rather than reported, so the monitor treatment - flip
+ * `disableActiveMonitoringBecauseOfScheduledMaintenanceEvent` and let the
+ * absence of new checks do the rest - does not transfer. A site under
+ * maintenance keeps reporting the truth about itself; what changes is who
+ * else has to listen:
+ *
+ *   - The site's OWN status is untouched. Someone looking at a unit during a
+ *     planned outage needs to see it is down.
+ *
+ *   - Its devices stop voting in its ANCESTORS' rollups, so the region above
+ *     it does not turn red for a cutover that was on the calendar. Because
+ *     the ancestor never records the outage, its uptime needs no correction
+ *     afterwards - which matters, since subtracting the window from the
+ *     region's denominator instead would also erase any GENUINE failure that
+ *     happened elsewhere in the region during the same hours.
+ *
+ *   - Its own uptime percentage excludes the window (see SiteUptimeUtil).
+ */
+/*
+ * Ongoing-maintenance lookups are cached per project for a few seconds.
+ *
+ * recomputeRollupForSite runs once per site per rollup, and the stale-rollup
+ * sweep walks up to five hundred sites in one pass; without this, each of
+ * those pays two queries to learn the same thing about the same project. The
+ * TTL is the entire staleness budget - a window that has just started takes
+ * at most this long to begin suppressing, which is nothing next to the
+ * five-minute sweep it is racing.
+ */
+const ONGOING_MAINTENANCE_CACHE_TTL_MS = 15 * 1000;
+/*
+ * Ceiling on cached projects. The map is keyed by project and each entry is a
+ * set of site ids, so on a large multi-tenant install it would otherwise grow
+ * to every project the process has ever rolled up and never shrink. Past this
+ * many entries the expired ones are swept before another is added.
+ */
+const MAX_CACHED_PROJECTS = 512;
+class NetworkSiteMaintenanceSuppression {
+    /*
+     * Ids of every site currently inside an ongoing maintenance window,
+     * expanded downward: attaching a Region yields the Region and every Market
+     * and Unit beneath it.
+     *
+     * Returns an empty set - without touching the hierarchy at all - when the
+     * project has no ongoing event, which is the overwhelmingly common case.
+     */
+    static async getSiteIdsUnderOngoingMaintenance(projectId) {
+        const cacheKey = projectId.toString();
+        const cached = this.cache.get(cacheKey);
+        const nowInMs = OneUptimeDate.getCurrentDate().getTime();
+        if (cached && cached.expiresAtInMs > nowInMs) {
+            return cached.siteIds;
+        }
+        const generationAtStart = this.generation;
+        let siteIds = new Set();
+        try {
+            siteIds = await this.resolveSiteIdsUnderOngoingMaintenance(projectId);
+        }
+        catch (err) {
+            /*
+             * A rollup must not fail because the maintenance lookup did. Falling
+             * back to "nothing is under maintenance" reproduces the behaviour from
+             * before this feature existed, which is the safe direction: a site may
+             * briefly turn red for planned work, rather than a real outage going
+             * unnoticed.
+             */
+            logger.error(`NetworkSiteMaintenanceSuppression: could not resolve ongoing maintenance for project ${cacheKey}; treating nothing as suppressed.`);
+            logger.error(err);
+            return new Set();
+        }
+        /*
+         * Someone invalidated while this query was in flight, so the answer in
+         * hand is already known to be stale. Serve it to this caller (it is no
+         * worse than what they would have got a moment earlier) but do not
+         * install it, so the next caller re-reads.
+         */
+        if (this.generation === generationAtStart) {
+            this.sweepIfCrowded();
+            this.cache.set(cacheKey, {
+                /*
+                 * Measured from AFTER the query, not before it. A slow query would
+                 * otherwise burn most of its own TTL before the entry was written.
+                 */
+                expiresAtInMs: OneUptimeDate.getCurrentDate().getTime() +
+                    ONGOING_MAINTENANCE_CACHE_TTL_MS,
+                siteIds: siteIds,
+            });
+        }
+        return siteIds;
+    }
+    // Drop expired entries once the map has grown past its ceiling.
+    static sweepIfCrowded() {
+        if (this.cache.size < MAX_CACHED_PROJECTS) {
+            return;
+        }
+        const nowInMs = OneUptimeDate.getCurrentDate().getTime();
+        for (const [key, entry] of this.cache) {
+            if (entry.expiresAtInMs <= nowInMs) {
+                this.cache.delete(key);
+            }
+        }
+        /*
+         * Everything was still live. Entries are a few seconds old at most, so
+         * clearing is cheap and bounded — far better than growing without limit.
+         */
+        if (this.cache.size >= MAX_CACHED_PROJECTS) {
+            this.cache.clear();
+        }
+    }
+    /*
+     * Drops the cached answer for a project. Called when an event changes
+     * state, so a window that has just started or ended takes effect on the
+     * very next rollup instead of up to a TTL later.
+     */
+    static invalidateCache(projectId) {
+        /*
+         * Bumped before the delete so a lookup that is mid-flight right now is
+         * refused its write-back too, not just future ones.
+         */
+        this.generation++;
+        if (!projectId) {
+            this.cache.clear();
+            return;
+        }
+        this.cache.delete(projectId.toString());
+    }
+    /*
+     * Every maintenance event in the project that OVERLAPS [windowStart,
+     * windowEnd), as the interval math needs it: the declared window plus the
+     * sites attached to it.
+     *
+     * The declared `startsAt`/`endsAt` are used rather than the state
+     * timeline's actual ongoing/ended transitions. They are what the user
+     * scheduled and what the event page shows, so the uptime number can be
+     * reconciled by hand against the calendar; deriving it from transitions
+     * would make the excluded interval depend on when a worker happened to run.
+     */
+    static async getMaintenanceEventWindows(data) {
+        const events = await ScheduledMaintenanceService.findBy({
+            query: {
+                projectId: data.projectId,
+                startsAt: QueryHelper.lessThanEqualTo(data.windowEnd),
+                endsAt: QueryHelper.greaterThanOrNull(data.windowStart),
+            },
+            select: {
+                _id: true,
+                startsAt: true,
+                endsAt: true,
+                networkSites: {
+                    _id: true,
+                },
+            },
+            limit: LIMIT_PER_PROJECT,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        return this.toEventWindows(events);
+    }
+    /*
+     * Shared shaping step: an event with no sites attached is dropped, because
+     * it excludes nothing and would only make the caller's loops longer.
+     */
+    static toEventWindows(events) {
+        const windows = [];
+        for (const event of events) {
+            if (!event.startsAt) {
+                continue;
+            }
+            const siteIds = (event.networkSites || [])
+                .map((site) => {
+                return site._id ? String(site._id) : "";
+            })
+                .filter((id) => {
+                return id.length > 0;
+            });
+            if (siteIds.length === 0) {
+                continue;
+            }
+            windows.push({
+                startsAt: event.startsAt,
+                endsAt: event.endsAt || null,
+                siteIds: siteIds,
+            });
+        }
+        return windows;
+    }
+    static async resolveSiteIdsUnderOngoingMaintenance(projectId) {
+        const ongoingEvents = await ScheduledMaintenanceService.findBy({
+            query: {
+                projectId: projectId,
+                currentScheduledMaintenanceState: {
+                    isOngoingState: true,
+                },
+            },
+            select: {
+                _id: true,
+                networkSites: {
+                    _id: true,
+                },
+            },
+            limit: LIMIT_PER_PROJECT,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        const attachedSiteIds = [];
+        const seen = new Set();
+        for (const event of ongoingEvents) {
+            for (const site of event.networkSites || []) {
+                if (!site._id) {
+                    continue;
+                }
+                const id = String(site._id);
+                if (seen.has(id)) {
+                    continue;
+                }
+                seen.add(id);
+                attachedSiteIds.push(new ObjectID(id));
+            }
+        }
+        if (attachedSiteIds.length === 0) {
+            return new Set();
+        }
+        return NetworkSiteService.getSubtreeSiteIds({
+            siteIds: attachedSiteIds,
+            projectId: projectId,
+        });
+    }
+}
+NetworkSiteMaintenanceSuppression.cache = new Map();
+/*
+ * Bumped by every invalidateCache. A lookup captures it BEFORE its query
+ * and refuses to write its answer back if it changed in the meantime.
+ *
+ * Without it the cache has a lost-update window exactly where it hurts: a
+ * maintenance event flips state and calls invalidateCache while an
+ * in-flight lookup is still awaiting the OLD answer, that lookup then
+ * stores the stale set, and the window it was supposed to start (or stop)
+ * is wrong for a further full TTL.
+ */
+NetworkSiteMaintenanceSuppression.generation = 0;
+export default NetworkSiteMaintenanceSuppression;
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [ObjectID]),
+    __metadata("design:returntype", Promise)
+], NetworkSiteMaintenanceSuppression, "getSiteIdsUnderOngoingMaintenance", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NetworkSiteMaintenanceSuppression, "getMaintenanceEventWindows", null);
+//# sourceMappingURL=NetworkSiteMaintenanceSuppression.js.map

package/build/dist/Server/Utils/NetworkSite/NetworkSiteMaintenanceSuppression.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NetworkSiteMaintenanceSuppression.js","sourceRoot":"","sources":["../../../../../Server/Utils/NetworkSite/NetworkSiteMaintenanceSuppression.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,kBAAkB,MAAM,mCAAmC,CAAC;AACnE,OAAO,2BAA2B,MAAM,4CAA4C,CAAC;AAErF,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,QAAQ,MAAM,yBAAyB,CAAC;AAC/C,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAChD,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAE3D,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,MAAM,MAAM,WAAW,CAAC;AAE/B;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH;;;;;;;;;GASG;AACH,MAAM,gCAAgC,GAAW,EAAE,GAAG,IAAI,CAAC;AAO3D;;;;;GAKG;AACH,MAAM,mBAAmB,GAAW,GAAG,CAAC;AAExC,MAAqB,iCAAiC;IAepD;;;;;;;OAOG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,iCAAiC,CACnD,SAAmB;QAEnB,MAAM,QAAQ,GAAW,SAAS,CAAC,QAAQ,EAAE,CAAC;QAC9C,MAAM,MAAM,GAA2B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChE,MAAM,OAAO,GAAW,aAAa,CAAC,cAAc,EAAE,CAAC,OAAO,EAAE,CAAC;QAEjE,IAAI,MAAM,IAAI,MAAM,CAAC,aAAa,GAAG,OAAO,EAAE,CAAC;YAC7C,OAAO,MAAM,CAAC,OAAO,CAAC;QACxB,CAAC;QAED,MAAM,iBAAiB,GAAW,IAAI,CAAC,UAAU,CAAC;QAElD,IAAI,OAAO,GAAgB,IAAI,GAAG,EAAU,CAAC;QAE7C,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,IAAI,CAAC,qCAAqC,CAAC,SAAS,CAAC,CAAC;QACxE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb;;;;;;eAMG;YACH,MAAM,CAAC,KAAK,CACV,wFAAwF,QAAQ,mCAAmC,CACpI,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClB,OAAO,IAAI,GAAG,EAAU,CAAC;QAC3B,CAAC;QAED;;;;;WAKG;QACH,IAAI,IAAI,CAAC,UAAU,KAAK,iBAAiB,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACvB;;;mBAGG;gBACH,aAAa,EACX,aAAa,CAAC,cAAc,EAAE,CAAC,OAAO,EAAE;oBACxC,gCAAgC;gBAClC,OAAO,EAAE,OAAO;aACjB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,gEAAgE;IACxD,MAAM,CAAC,cAAc;QAC3B,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,mBAAmB,EAAE,CAAC;YAC1C,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAW,aAAa,CAAC,cAAc,EAAE,CAAC,OAAO,EAAE,CAAC;QAEjE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,aAAa,IAAI,OAAO,EAAE,CAAC;gBACnC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED;;;WAGG;QACH,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,mBAAmB,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,eAAe,CAAC,SAAgC;QAC5D;;;WAGG;QACH,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED;;;;;;;;;;OAUG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,IAI9C;QACC,MAAM,MAAM,GACV,MAAM,2BAA2B,CAAC,MAAM,CAAC;YACvC,KAAK,EAAE;gBACL,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,QAAQ,EAAE,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC;gBACrD,MAAM,EAAE,WAAW,CAAC,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC;aACxD;YACD,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI;gBACT,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE;oBACZ,GAAG,EAAE,IAAI;iBACV;aACF;YACD,KAAK,EAAE,iBAAiB;YACxB,IAAI,EAAE,CAAC;YACP,KAAK,EAAE;gBACL,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QAEL,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,cAAc,CAC1B,MAAmC;QAEnC,MAAM,OAAO,GAAkC,EAAE,CAAC;QAElD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACpB,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAkB,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC;iBACtD,GAAG,CAAC,CAAC,IAAkC,EAAE,EAAE;gBAC1C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1C,CAAC,CAAC;iBACD,MAAM,CAAC,CAAC,EAAU,EAAE,EAAE;gBACrB,OAAO,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;YAEL,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,SAAS;YACX,CAAC;YAED,OAAO,CAAC,IAAI,CAAC;gBACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,IAAI;gBAC5B,OAAO,EAAE,OAAO;aACjB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,qCAAqC,CACxD,SAAmB;QAEnB,MAAM,aAAa,GACjB,MAAM,2BAA2B,CAAC,MAAM,CAAC;YACvC,KAAK,EAAE;gBACL,SAAS,EAAE,SAAS;gBACpB,gCAAgC,EAAE;oBAChC,cAAc,EAAE,IAAI;iBACrB;aACF;YACD,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI;gBACT,YAAY,EAAE;oBACZ,GAAG,EAAE,IAAI;iBACV;aACF;YACD,KAAK,EAAE,iBAAiB;YACxB,IAAI,EAAE,CAAC;YACP,KAAK,EAAE;gBACL,MAAM,EAAE,IAAI;aACb;SACF,CAAC,CAAC;QAEL,MAAM,eAAe,GAAoB,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAgB,IAAI,GAAG,EAAU,CAAC;QAE5C,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;YAClC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACd,SAAS;gBACX,CAAC;gBACD,MAAM,EAAE,GAAW,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpC,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBACjB,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACb,eAAe,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,GAAG,EAAU,CAAC;QAC3B,CAAC;QAED,OAAO,kBAAkB,CAAC,iBAAiB,CAAC;YAC1C,OAAO,EAAE,eAAe;YACxB,SAAS,EAAE,SAAS;SACrB,CAAC,CAAC;IACL,CAAC;;AAvPc,uCAAK,GAA4B,IAAI,GAAG,EAAE,CAAC;AAE1D;;;;;;;;;GASG;AACY,4CAAU,GAAW,CAAC,CAAC;eAbnB,iCAAiC;AAwBhC;IADnB,WAAW,EAAE;;qCAED,QAAQ;;gFAoDpB;AAwDmB;IADnB,WAAW,EAAE;;;;yEA6Bb"}