@oneuptime/common 14.0.0 → 14.0.2

package/build/dist/Server/Services/MonitorStatusTimelineService.js

@@ -25,6 +25,10 @@ import MonitorFeedService from "./MonitorFeedService";
 import { MonitorFeedEventType } from "../../Models/DatabaseModels/MonitorFeed";
 import MonitorStatusService from "./MonitorStatusService";
 import ServerException from "../../Types/Exception/ServerException";
+import { MONITOR_UPTIME_HISTORY_DAYS, MONITOR_UPTIME_ROLLING_WINDOWS, MonitorUptimeWindowKey, } from "../../Types/Monitor/MonitorUptimeSummary";
+import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
+import MonitorUptimeSummaryUtil from "../../Utils/Monitor/MonitorUptimeSummaryUtil";
+import UptimeDailyAggregateUtil from "../../Utils/StatusPage/UptimeDailyAggregateUtil";
 /*
  * Thrown by onBeforeCreate when the incoming status is the same as the status of
  * the row immediately before it. Probe ingest call sites (Utils/Monitor/MonitorStatusTimeline
@@ -76,6 +80,14 @@ export class Service extends DatabaseService {
      *    and double-count that span. This mirrors what
      *    UptimeUtil.getMonitorEventsForId already does client-side.
      *
+     *    "Next" is `ORDER BY startsAt, endsAt NULLS LAST`. With startsAt alone
+     *    an exact tie is left in whatever order the plan produces, and when the
+     *    open row comes first its LEAD is the tied row's start - its own start -
+     *    so the monitor's CURRENT status is cut to zero length and today reads
+     *    as no data. Putting the open row last on a tie is what
+     *    getRollingUptimeTotals and UptimeUtil.compareTimelinesChronologically
+     *    do, so all three agree on the same rows.
+     *
      * 2. Periods are split into days ARITHMETICALLY (`generate_series` over the
      *    days a period touches) and then hash-aggregated, rather than joined
      *    against a bucket grid. Both give the same answer; the join does not
@@ -115,6 +127,7 @@ export class Service extends DatabaseService {
             monitors: [],
             isComplete: true,
             completeFrom: null,
+            timezone: data.timezone || "UTC",
         };
         if (data.monitorIds.length === 0) {
             return empty;
@@ -143,7 +156,8 @@ export class Service extends DatabaseService {
                  COALESCE(
                    t."endsAt",
                    LEAD(t."startsAt") OVER (
-                     PARTITION BY t."monitorId" ORDER BY t."startsAt"
+                     PARTITION BY t."monitorId"
+                     ORDER BY t."startsAt", t."endsAt" NULLS LAST
                    ),
                    p.eff_end
                  ),
@@ -216,7 +230,7 @@ export class Service extends DatabaseService {
             data.endDate,
             data.timezone || "UTC",
         ]);
-        return Service.toUptimeDailyAggregate(rows);
+        return Service.toUptimeDailyAggregate(rows, data.timezone || "UTC");
     }
     /*
      * Exported shape-only step, so the bucketing can be tested without a
@@ -224,7 +238,13 @@ export class Service extends DatabaseService {
      * comes back as a single row with a NULL status and zero seconds, which is
      * what makes a no-data day distinguishable from a day spent up.
      */
-    static toUptimeDailyAggregate(rows) {
+    static toUptimeDailyAggregate(rows, 
+    /*
+     * The zone the SQL cut the day buckets in. It travels with the buckets
+     * so a client draws its bars on the same day boundaries - see
+     * UptimeDailyAggregate.timezone.
+     */
+    timezone = "UTC") {
         const byMonitor = new Map();
         for (const row of rows) {
             let buckets = byMonitor.get(row.monitorId);
@@ -277,6 +297,505 @@ export class Service extends DatabaseService {
             monitors: monitors,
             isComplete: true,
             completeFrom: null,
+            timezone: timezone,
+        };
+    }
+    /**
+     * How long a SET of monitors was down over [startDate, endDate]: the time
+     * at least one of them was in a downtime status, and the time at least one
+     * of them was recorded at all, so the caller can take a percentage.
+     *
+     * WHY THIS EXISTS
+     *
+     * The status page report's page and group downtime, and the uptime of a
+     * monitor-group resource in the report and the uptime endpoint, merged the
+     * rows of getMonitorStatusTimelineForStatusPage. Those arrive under one
+     * LIMIT_MAX (10,000) across every monitor on the page, newest first, and
+     * status.chainflip.io matched 255,733 rows in 60 days - so every merged
+     * figure covered only the newest few days. getDailyUptimeAggregate has no
+     * cap, but it sums each monitor on its own, and per-monitor sums cannot
+     * say when at least one monitor was down: two monitors down over the same
+     * hour are one hour of downtime for their group, and over different hours
+     * two.
+     *
+     * WHAT IT COUNTS
+     *
+     * The rows, and where each one ends, are exactly getDailyUptimeAggregate's
+     * src CTE: the same inclusive overlap predicate, deleted rows ignored,
+     * everything clipped to [startDate, LEAST(endDate, now())], and an open row
+     * capped at the next row's start, `ORDER BY startsAt, endsAt NULLS LAST`.
+     * For a single monitor the two agree exactly, unless two of its own rows
+     * overlap (deleting a row from the middle of a timeline leaves that, see
+     * getRollingUptimeTotals): the aggregate counts the overlap twice, and a
+     * union once.
+     *
+     * Downtime statuses are picked out AFTER that LEAD, never before it.
+     * Filtered first, an open Offline row would run on to the next Offline
+     * row's start, across the Operational rows in between.
+     *
+     * THE MERGE
+     *
+     * Every monitor's periods are taken in start order, and each period adds
+     * only what it reaches past the latest end of the periods before it (a
+     * running MAX(end)). One that starts after that point opens a new island
+     * and adds its whole length; one that starts inside it adds the part that
+     * sticks out, or nothing. The sum is the length of the union, which is
+     * what numbering the islands and adding them up would give, without the
+     * second window pass - and it does not depend on how tied periods are
+     * ordered. Coverage and downtime are merged in the same pass; downtime's
+     * running end is taken over downtime periods only.
+     *
+     * The union is the intended meaning of a merged figure. The rows used to
+     * be merged by UptimeUtil.getNonOverlappingMonitorEvents, a priority
+     * heuristic that undercounts: a later-starting, longer-running Operational
+     * row of another monitor truncates an earlier Offline row. With A Offline
+     * from 00:00 to 10:00 and B Operational from 02:00 to 20:00 it counts two
+     * hours down, where A alone was down for ten.
+     *
+     * Seconds are double precision and never rounded, as in the aggregate.
+     *
+     * Checked against Postgres with randomised timelines - orphaned open rows,
+     * backfill ties, gaps, overlaps, deleted, corrupt and future rows - and a
+     * sweep-line union written separately: exact in every one of 3,300
+     * windows. It reads the same rows the aggregate reads for those monitors
+     * and sorts their periods once instead of splitting them into days; on a
+     * 290k-row page with a monitor flapping every 20 s it ran in about two
+     * thirds of the aggregate's time.
+     */
+    async getMergedDowntimeSeconds(data) {
+        if (data.monitorIds.length === 0) {
+            return {
+                coveredSeconds: 0,
+                downtimeSeconds: 0,
+            };
+        }
+        const sql = `
+      WITH params AS (
+        SELECT $2::timestamptz AS win_start,
+               LEAST($3::timestamptz, now()) AS eff_end
+      ),
+      src AS (
+        SELECT t."monitorStatusId" AS status_id,
+               GREATEST(t."startsAt", p.win_start) AS s,
+               LEAST(
+                 COALESCE(
+                   t."endsAt",
+                   LEAD(t."startsAt") OVER (
+                     PARTITION BY t."monitorId"
+                     ORDER BY t."startsAt", t."endsAt" NULLS LAST
+                   ),
+                   p.eff_end
+                 ),
+                 p.eff_end
+               ) AS e
+        FROM "MonitorStatusTimeline" t
+        CROSS JOIN params p
+        WHERE t."monitorId" = ANY($1::uuid[])
+          AND t."deletedAt" IS NULL
+          AND t."startsAt" <= p.eff_end
+          AND (t."endsAt" >= p.win_start OR t."endsAt" IS NULL)
+      ),
+      periods AS (
+        SELECT s,
+               e,
+               status_id = ANY($4::uuid[]) AS is_downtime
+        FROM src
+        WHERE e > s
+      ),
+      reach AS (
+        SELECT s,
+               e,
+               is_downtime,
+               MAX(e) OVER earlier AS covered_until,
+               MAX(e) FILTER (WHERE is_downtime) OVER earlier AS down_until
+        FROM periods
+        WINDOW earlier AS (
+          ORDER BY s, e
+          ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
+        )
+      )
+      SELECT COALESCE(
+               SUM(
+                 GREATEST(
+                   EXTRACT(EPOCH FROM (e - GREATEST(s, COALESCE(covered_until, s)))),
+                   0
+                 )
+               ),
+               0
+             )::double precision AS "coveredSeconds",
+             COALESCE(
+               SUM(
+                 GREATEST(
+                   EXTRACT(EPOCH FROM (e - GREATEST(s, COALESCE(down_until, s)))),
+                   0
+                 )
+               ) FILTER (WHERE is_downtime),
+               0
+             )::double precision AS "downtimeSeconds"
+      FROM reach
+    `;
+        const rows = await this.getRepository().manager.query(sql, [
+            data.monitorIds.map((id) => {
+                return id.toString();
+            }),
+            data.startDate,
+            data.endDate,
+            data.downtimeMonitorStatusIds.map((id) => {
+                return id.toString();
+            }),
+        ]);
+        return Service.toMergedDowntimeTotals(rows);
+    }
+    /*
+     * Shape-only step, public so it can be tested without a database. The SQL
+     * always returns one row; a missing row, or a value that is not a
+     * positive number, reads as zero.
+     */
+    static toMergedDowntimeTotals(rows) {
+        var _a, _b;
+        const coveredSeconds = Number((_a = rows[0]) === null || _a === void 0 ? void 0 : _a.coveredSeconds);
+        const downtimeSeconds = Number((_b = rows[0]) === null || _b === void 0 ? void 0 : _b.downtimeSeconds);
+        return {
+            coveredSeconds: coveredSeconds > 0 ? coveredSeconds : 0,
+            downtimeSeconds: downtimeSeconds > 0 ? downtimeSeconds : 0,
+        };
+    }
+    /**
+     * Per-status seconds and covered seconds for ONE monitor over several
+     * windows that all end at `endDate` - the monitor overview's rolling 24h,
+     * 7d and 30d figures - read in one statement.
+     *
+     * WHY IT IS NOT getDailyUptimeAggregate PER WINDOW
+     *
+     * It used to be: one aggregate call per window, each summed from its own
+     * day buckets. That SQL can only hand the (monitorId, startsAt) index an
+     * upper bound (`startsAt <= end`). Its lower bound is on `endsAt`, which
+     * that index does not hold, so every call - the 24 hour one included -
+     * read every row the monitor has ever had. Nothing deletes old timeline
+     * rows, and the overview reloads the summary as the monitor changes
+     * status, so a monitor that has flapped for two years paid for three extra
+     * full-history reads per reload, next to the one the bars already make.
+     *
+     * WHAT IT COUNTS
+     *
+     * Per window, the rows and how each is clipped are exactly
+     * getDailyUptimeAggregate's src CTE: the same inclusive overlap predicate,
+     * deleted rows ignored, the end clamped to now(), and an open row capped at
+     * the start of the next row that overlaps THE SAME WINDOW. That last part
+     * is why the rows are fanned out per window before the LEAD rather than
+     * given one LEAD: an orphaned open row followed by a row that closed before
+     * a window started runs on to the first row inside that window, as it
+     * would in that window's own aggregate call. Seconds are summed as double
+     * precision and never rounded. There is no day split: splitting a period
+     * into days and adding the days back up gives the same total, and a
+     * rolling window has no day boundaries to report.
+     *
+     * On an exact startsAt tie a closed row sorts before an open one. The
+     * aggregate leaves ties in whatever order the plan produces; putting the
+     * open row last means the zero-length row that closePrecedingStatusTimeline
+     * leaves on a backfill tie cannot cut the open row short.
+     *
+     * The fan-out is partitioned and grouped by each window's ordinal, not its
+     * key. Sorting tens of thousands of rows on a collated text key was most of
+     * the statement's time.
+     *
+     * THE LOWER BOUND
+     *
+     * The rows are read from the start of the newest live row that began
+     * before the widest window (rows tied with it included), plus every open
+     * row older than that. The start is one backward step down (monitorId,
+     * startsAt), and the read is a range scan on the same index, so its cost
+     * follows the window rather than the monitor's age. The older open rows
+     * are orphans a racing writer left behind. The newest of them runs on until
+     * the next row that overlaps a window, so they cannot be dropped. The
+     * planner finds them with a BitmapAnd of the (monitorId, startsAt) and
+     * (endsAt) indexes, which reads index pages but none of the old rows.
+     *
+     * Checked against a synthetic 800k-row table, with a 200k-row monitor,
+     * orphans, gaps, ties and deleted rows. Per window and status, this
+     * agreed with getDailyUptimeAggregate to within 1e-9 s, in four zones. One
+     * statement read about 1.7k buffers, where each of the three aggregate
+     * calls it replaces read 10k.
+     *
+     * That row set is exactly what the aggregate reads while no closed row
+     * ends after a later row starts. The write paths keep that: create closes
+     * the preceding row at the new row's start, and a backfilled row gets the
+     * next row's start as its end. The exception is deleting a row from the
+     * middle of a timeline (onBeforeDelete). That moves the preceding row's
+     * end to the next row's old start and the next row's start back to the
+     * deleted row's start, so the two overlap across the deleted span. Where
+     * that overlap straddles the widest window's start, the aggregate counts
+     * it twice, reporting more than 100% coverage, and this counts it once,
+     * from the newer row.
+     */
+    async getRollingUptimeTotals(data) {
+        if (data.windows.length === 0) {
+            return [];
+        }
+        const sql = `
+      WITH params AS (
+        SELECT LEAST($2::timestamptz, now()) AS eff_end
+      ),
+      windows AS (
+        SELECT w.window_key, w.win_start, w.window_no
+        FROM UNNEST($3::text[], $4::timestamptz[])
+             WITH ORDINALITY AS w(window_key, win_start, window_no)
+      ),
+      widest AS (
+        SELECT MIN(win_start) AS win_start FROM windows
+      ),
+      lower_bound AS (
+        SELECT COALESCE(
+                 (
+                   SELECT MAX(t."startsAt")
+                   FROM "MonitorStatusTimeline" t
+                   WHERE t."monitorId" = $1::uuid
+                     AND t."deletedAt" IS NULL
+                     AND t."startsAt" < wd.win_start
+                 ),
+                 wd.win_start
+               ) AS starts_from
+        FROM widest wd
+      ),
+      src_rows AS (
+        SELECT t."monitorStatusId" AS status_id,
+               t."startsAt" AS starts_at,
+               t."endsAt" AS ends_at
+        FROM "MonitorStatusTimeline" t
+        CROSS JOIN params p
+        CROSS JOIN widest wd
+        WHERE t."monitorId" = $1::uuid
+          AND t."deletedAt" IS NULL
+          AND t."startsAt" >= (SELECT starts_from FROM lower_bound)
+          AND t."startsAt" <= p.eff_end
+          AND (t."endsAt" >= wd.win_start OR t."endsAt" IS NULL)
+        UNION ALL
+        SELECT t."monitorStatusId" AS status_id,
+               t."startsAt" AS starts_at,
+               t."endsAt" AS ends_at
+        FROM "MonitorStatusTimeline" t
+        CROSS JOIN params p
+        WHERE t."monitorId" = $1::uuid
+          AND t."deletedAt" IS NULL
+          AND t."endsAt" IS NULL
+          AND t."startsAt" < (SELECT starts_from FROM lower_bound)
+          AND t."startsAt" <= p.eff_end
+      ),
+      src AS (
+        SELECT w.window_no,
+               r.status_id,
+               GREATEST(r.starts_at, w.win_start) AS s,
+               LEAST(
+                 COALESCE(
+                   r.ends_at,
+                   LEAD(r.starts_at) OVER (
+                     PARTITION BY w.window_no
+                     ORDER BY r.starts_at, r.ends_at NULLS LAST
+                   ),
+                   p.eff_end
+                 ),
+                 p.eff_end
+               ) AS e
+        FROM src_rows r
+        CROSS JOIN windows w
+        CROSS JOIN params p
+        WHERE r.ends_at >= w.win_start OR r.ends_at IS NULL
+      ),
+      agg AS (
+        SELECT window_no,
+               status_id,
+               SUM(EXTRACT(EPOCH FROM (e - s)))::double precision AS seconds
+        FROM src
+        WHERE e > s
+        GROUP BY window_no, status_id
+      )
+      SELECT w.window_key AS "windowKey",
+             GREATEST(EXTRACT(EPOCH FROM (p.eff_end - w.win_start)), 0)
+               ::double precision AS "windowSeconds",
+             a.status_id AS "monitorStatusId",
+             COALESCE(a.seconds, 0)::double precision AS "seconds"
+      FROM windows w
+      CROSS JOIN params p
+      LEFT JOIN agg a ON a.window_no = w.window_no
+      ORDER BY w.window_no, a.status_id
+    `;
+        const rows = await this.getRepository().manager.query(sql, [
+            data.monitorId.toString(),
+            data.endDate,
+            data.windows.map((window) => {
+                return window.key;
+            }),
+            data.windows.map((window) => {
+                return window.startDate.toISOString();
+            }),
+        ]);
+        return Service.toRollingUptimeTotals({
+            rows: rows,
+            windows: data.windows,
+            endDate: data.endDate,
+        });
+    }
+    /*
+     * Shape-only step, public so it can be tested without a database. One
+     * row per (window, status); a window nothing overlapped comes back as a
+     * single row with a NULL status and zero seconds, so it keeps its length
+     * and reads as "No data" rather than disappearing. The totals come back in
+     * the order the windows were asked for.
+     */
+    static toRollingUptimeTotals(data) {
+        return data.windows.map((window) => {
+            let windowSeconds = 0;
+            let coveredSeconds = 0;
+            const statusDurations = [];
+            for (const row of data.rows) {
+                if (row.windowKey !== window.key) {
+                    continue;
+                }
+                windowSeconds = Number(row.windowSeconds);
+                const seconds = Number(row.seconds);
+                // The LEFT JOIN's "nothing overlapped" row is not a status.
+                if (!row.monitorStatusId || !(seconds > 0)) {
+                    continue;
+                }
+                coveredSeconds += seconds;
+                statusDurations.push({
+                    monitorStatusId: new ObjectID(row.monitorStatusId),
+                    seconds: seconds,
+                });
+            }
+            /*
+             * A rolling window is one bucket that starts mid-day. Summing it
+             * through the same function as the 90d window keeps the ordering of
+             * statusDurations and the handling of bad numbers identical.
+             */
+            return MonitorUptimeSummaryUtil.sumBuckets({
+                key: window.key,
+                buckets: [
+                    {
+                        bucketStart: window.startDate,
+                        bucketEnd: data.endDate,
+                        daySeconds: windowSeconds,
+                        coveredSeconds: coveredSeconds,
+                        statusDurations: statusDurations,
+                    },
+                ],
+                startDate: window.startDate,
+                endDate: data.endDate,
+            });
+        });
+    }
+    /**
+     * The uptime history of ONE monitor for the monitor overview: 90 calendar
+     * day bars, rolling 24h / 7d / 30d windows, a 90d window, and the
+     * project's statuses so the page can tell downtime from uptime without a
+     * MonitorStatus request of its own.
+     *
+     * It reads as root and does no authorisation. The route in MonitorAPI
+     * decides who may see the monitor's history before calling this.
+     *
+     * TWO READS OF THE TIMELINE
+     *
+     * The bars are one getDailyUptimeAggregate call. A rolling 24 hours starts
+     * in the middle of a day, which the midnight-aligned bars cannot express,
+     * so the rolling windows come from getRollingUptimeTotals, which reads the
+     * rows once for all three (see there for why that is not one aggregate
+     * call per window). The 90d figure is summed from the bars instead, so it
+     * always agrees with the strip it sits next to.
+     *
+     * The time zone only moves where the day boundaries fall. The rolling
+     * windows are exact second counts ending at now, whatever the zone.
+     */
+    async getMonitorUptimeSummary(data) {
+        const now = data.now;
+        /*
+         * Calendar arithmetic in the caller's zone, then local midnight. Taking
+         * 89 x 24 hours instead would land on the wrong calendar day across a DST
+         * change and hand back 89 or 91 bars.
+         */
+        const barsStartDate = OneUptimeDate.getStartOfDay(OneUptimeDate.addRemoveDays(now, -(MONITOR_UPTIME_HISTORY_DAYS - 1), data.timezone), data.timezone);
+        const rollingWindows = MONITOR_UPTIME_ROLLING_WINDOWS.map((window) => {
+            return {
+                key: window.key,
+                startDate: OneUptimeDate.addRemoveSeconds(now, -window.seconds),
+            };
+        });
+        const [barAggregate, rollingTotals, statuses] = await Promise.all([
+            this.getDailyUptimeAggregate({
+                monitorIds: [data.monitorId],
+                startDate: barsStartDate,
+                endDate: now,
+                timezone: data.timezone,
+            }),
+            this.getRollingUptimeTotals({
+                monitorId: data.monitorId,
+                windows: rollingWindows,
+                endDate: now,
+            }),
+            /*
+             * Root, scoped to the project, and limited to columns every one of
+             * which is canReadOnRelationQuery. The route has already proved the
+             * caller can read this monitor's timeline, and a timeline read
+             * exposes exactly these columns through its monitorStatus relation,
+             * so nothing is disclosed that the CRUD read would not show.
+             */
+            MonitorStatusService.findBy({
+                query: {
+                    projectId: data.projectId,
+                },
+                select: {
+                    _id: true,
+                    name: true,
+                    color: true,
+                    isOperationalState: true,
+                    isOfflineState: true,
+                    priority: true,
+                },
+                sort: {
+                    priority: SortOrder.Ascending,
+                },
+                limit: LIMIT_PER_PROJECT,
+                skip: 0,
+                props: {
+                    isRoot: true,
+                },
+            }),
+        ]);
+        const barBuckets = UptimeDailyAggregateUtil.getBucketsForMonitor(barAggregate, data.monitorId);
+        const windows = [...rollingTotals];
+        windows.push(MonitorUptimeSummaryUtil.sumBuckets({
+            key: MonitorUptimeWindowKey.Last90Days,
+            buckets: barBuckets,
+            startDate: barsStartDate,
+            endDate: now,
+        }));
+        const summaryStatuses = [];
+        for (const status of statuses) {
+            if (!status.id) {
+                continue;
+            }
+            summaryStatuses.push({
+                id: status.id,
+                name: status.name || "",
+                color: status.color ? status.color.toString() : "",
+                isOperationalState: status.isOperationalState === true,
+                isOfflineState: status.isOfflineState === true,
+                priority: typeof status.priority === "number" &&
+                    Number.isFinite(status.priority)
+                    ? status.priority
+                    : null,
+            });
+        }
+        return {
+            monitorId: data.monitorId,
+            timezone: data.timezone,
+            generatedAt: now,
+            startDate: barsStartDate,
+            endDate: now,
+            buckets: barBuckets,
+            windows: windows,
+            isComplete: barAggregate.isComplete,
+            completeFrom: barAggregate.completeFrom,
+            statuses: summaryStatuses,
         };
     }
     async create(createBy) {
@@ -1003,6 +1522,24 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "getDailyUptimeAggregate", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getMergedDowntimeSeconds", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getRollingUptimeTotals", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getMonitorUptimeSummary", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),