@oneuptime/common 11.5.11 → 11.5.13

package/Server/Services/MonitorStatusTimelineService.ts

@@ -18,6 +18,25 @@ import MonitorFeedService from "./MonitorFeedService";
 import { MonitorFeedEventType } from "../../Models/DatabaseModels/MonitorFeed";
 import MonitorStatus from "../../Models/DatabaseModels/MonitorStatus";
 import MonitorStatusService from "./MonitorStatusService";
+import ServerException from "../../Types/Exception/ServerException";
+
+/*
+ * Thrown by onBeforeCreate when the incoming status is the same as the status of
+ * the row immediately before it. Probe ingest call sites (Utils/Monitor/MonitorStatusTimeline
+ * and Utils/Monitor/MonitorResource) match on this exact message to treat the
+ * duplicate as an idempotent no-op, so the text must not change.
+ */
+export const MONITOR_STATUS_SAME_AS_PREVIOUS_ERROR_MESSAGE: string =
+  "Monitor Status cannot be same as previous status.";
+
+/*
+ * Thrown by create() when the per-monitor mutex cannot be acquired. The timeline
+ * write is refused rather than performed unlocked - see the comment on the lock
+ * acquisition below. Probe ingest call sites match on this to log and skip
+ * instead of failing the whole ingest run.
+ */
+export const MONITOR_STATUS_TIMELINE_LOCK_ERROR_MESSAGE: string =
+  "Could not acquire the monitor status timeline lock for this monitor.";
 
 export class Service extends DatabaseService<MonitorStatusTimeline> {
   public constructor() {
@@ -25,13 +44,35 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
   }
 
   @CaptureSpan()
-  protected override async onBeforeCreate(
+  public override async create(
     createBy: CreateBy<MonitorStatusTimeline>,
-  ): Promise<OnCreate<MonitorStatusTimeline>> {
-    if (!createBy.data.monitorId) {
-      throw new BadDataException("monitorId is null");
+  ): Promise<MonitorStatusTimeline> {
+    /*
+     * The per-monitor mutex is owned here, around the whole create, rather than
+     * inside onBeforeCreate. DatabaseService.create() invokes onBeforeCreate and
+     * then runs validation, permission checks and the INSERT before it ever
+     * reaches onCreateSuccess, all OUTSIDE any try/catch this service can hook
+     * (onCreateError never fires for a throw raised before the INSERT). So a
+     * mutex acquired in onBeforeCreate and released in onCreateSuccess leaks on
+     * any throw in between, and a leaked redis-semaphore mutex never expires - it
+     * keeps refreshing its own Redis key for the life of the process, which would
+     * block every later create for that monitor until acquireTimeout. Holding it
+     * in a try/finally here releases it on every path.
+     *
+     * The critical section that must be serialized per monitor spans reading the
+     * predecessor row (onBeforeCreate) through closing it (onCreateSuccess), so
+     * the lock is held across the entire super.create(), not just one hook.
+     */
+    if (createBy.props.ignoreHooks || !createBy.data.monitorId) {
+      // No predecessor bookkeeping runs on these paths, so no serialization is needed.
+      return await super.create(createBy);
     }
 
+    const logAttributes: LogAttributes = {
+      projectId: createBy.data.projectId?.toString(),
+      monitorId: createBy.data.monitorId?.toString(),
+    } as LogAttributes;
+
     let mutex: SemaphoreMutex | null = null;
 
     try {
@@ -40,12 +81,79 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
         namespace: "MonitorStatusTimeline.create",
       });
     } catch (e) {
-      logger.error(e, {
-        projectId: createBy.data.projectId?.toString(),
-        monitorId: createBy.data.monitorId?.toString(),
-      } as LogAttributes);
+      /*
+       * Fail closed. This used to fall through and INSERT UNLOCKED, which let two
+       * concurrent writers resolve the same predecessor row, both pass the
+       * same-as-previous check, and both INSERT a status row milliseconds apart.
+       * Only the later row is ever closed (the next writer resolves its
+       * predecessor with ORDER BY startsAt DESC LIMIT 1), so the earlier row is
+       * orphaned with endsAt = NULL permanently and is read back as unbounded
+       * downtime. Refusing the write is strictly safer: the monitor keeps its
+       * current status and the next probe result for the same monitor
+       * re-evaluates the same criteria and recreates the status change.
+       */
+      logger.error(e, logAttributes);
+      throw new ServerException(MONITOR_STATUS_TIMELINE_LOCK_ERROR_MESSAGE);
+    }
+
+    let createdItem: MonitorStatusTimeline;
+
+    try {
+      createdItem = await super.create(createBy);
+    } finally {
+      await this.releaseMutex(mutex, logAttributes);
     }
 
+    /*
+     * The feed item and its workspace notification run AFTER the mutex is
+     * released: they can involve third-party HTTP (Slack/Teams) with unbounded
+     * latency, and holding the per-monitor lock across them would block every
+     * concurrent status write for this monitor until acquireTimeout - turning
+     * one slow webhook into refused status transitions. Only the predecessor
+     * read -> INSERT -> predecessor close needs the lock, and all of that has
+     * completed by this point. (The pre-fail-closed code released the lock at
+     * this same boundary, before the feed block.)
+     */
+    await this.createStatusChangeFeedItem(createdItem, createBy);
+
+    return createdItem;
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<MonitorStatusTimeline>,
+  ): Promise<OnCreate<MonitorStatusTimeline>> {
+    if (!createBy.data.monitorId) {
+      throw new BadDataException("monitorId is null");
+    }
+
+    const logAttributes: LogAttributes = {
+      projectId: createBy.data.projectId?.toString(),
+      monitorId: createBy.data.monitorId?.toString(),
+    } as LogAttributes;
+
+    /*
+     * The per-monitor mutex that serializes this read-modify-write is acquired
+     * and released in create() (see the comment there); it is held for the whole
+     * duration of this hook.
+     */
+    return await this.buildOnCreate(
+      createBy,
+      createBy.data.monitorId,
+      logAttributes,
+    );
+  }
+
+  /*
+   * Body of onBeforeCreate, split out to keep the null-narrowing of monitorId in
+   * one place: it is passed in already narrowed so it can never reach a query as
+   * undefined, which would widen the query to every monitor.
+   */
+  private async buildOnCreate(
+    createBy: CreateBy<MonitorStatusTimeline>,
+    monitorId: ObjectID,
+    logAttributes: LogAttributes,
+  ): Promise<OnCreate<MonitorStatusTimeline>> {
     if (!createBy.data.startsAt) {
       createBy.data.startsAt = OneUptimeDate.getCurrentDate();
     }
@@ -85,7 +193,7 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
 
     const stateBeforeThis: MonitorStatusTimeline | null = await this.findOneBy({
       query: {
-        monitorId: createBy.data.monitorId,
+        monitorId: monitorId,
         startsAt: QueryHelper.lessThanEqualTo(createBy.data.startsAt),
       },
       sort: {
@@ -101,14 +209,8 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
       },
     });
 
-    logger.debug("State Before this", {
-      projectId: createBy.data.projectId?.toString(),
-      monitorId: createBy.data.monitorId?.toString(),
-    } as LogAttributes);
-    logger.debug(stateBeforeThis, {
-      projectId: createBy.data.projectId?.toString(),
-      monitorId: createBy.data.monitorId?.toString(),
-    } as LogAttributes);
+    logger.debug("State Before this", logAttributes);
+    logger.debug(stateBeforeThis, logAttributes);
 
     // If this is the first state, then do not notify the owner.
     if (!stateBeforeThis) {
@@ -126,15 +228,26 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
         stateBeforeThis.monitorStatusId.toString() ===
         monitorStatusId.toString()
       ) {
+        /*
+         * Logged above debug on purpose. This is the exact point at which an
+         * orphaned (endsAt = NULL) row would have been created had the mutex
+         * fallen through unlocked, so it is the signal to alert on: a sustained
+         * rate here means two writers are racing for the same monitor.
+         */
+        logger.warn(
+          `MonitorStatusTimeline: rejecting duplicate status ${monitorStatusId.toString()} for monitor ${monitorId.toString()}. The preceding status starting at ${stateBeforeThis.startsAt?.toString()} is already this status.`,
+          logAttributes,
+        );
+
         throw new BadDataException(
-          "Monitor Status cannot be same as previous status.",
+          MONITOR_STATUS_SAME_AS_PREVIOUS_ERROR_MESSAGE,
         );
       }
     }
 
     const stateAfterThis: MonitorStatusTimeline | null = await this.findOneBy({
       query: {
-        monitorId: createBy.data.monitorId,
+        monitorId: monitorId,
         startsAt: QueryHelper.greaterThan(createBy.data.startsAt),
       },
       sort: {
@@ -170,30 +283,108 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
       }
     }
 
-    logger.debug("State After this", {
-      projectId: createBy.data.projectId?.toString(),
-      monitorId: createBy.data.monitorId?.toString(),
-    } as LogAttributes);
-    logger.debug(stateAfterThis, {
-      projectId: createBy.data.projectId?.toString(),
-      monitorId: createBy.data.monitorId?.toString(),
-    } as LogAttributes);
+    logger.debug("State After this", logAttributes);
+    logger.debug(stateAfterThis, logAttributes);
 
     return {
       createBy,
       carryForward: {
         statusTimelineBeforeThisStatus: stateBeforeThis || null,
         statusTimelineAfterThisStatus: stateAfterThis || null,
-        mutex: mutex,
       },
     };
   }
 
+  /*
+   * Releases the per-monitor mutex. Never throws: a failed release must not mask
+   * the error we are already unwinding, and must not fail an otherwise successful
+   * create. Semaphore.release stops the refresh interval before it talks to
+   * Redis, so even if the Redis call fails the key expires on its own lockTimeout.
+   */
+  private async releaseMutex(
+    mutex: SemaphoreMutex | null | undefined,
+    logAttributes: LogAttributes,
+  ): Promise<void> {
+    if (!mutex) {
+      return;
+    }
+
+    try {
+      await Semaphore.release(mutex);
+    } catch (err) {
+      logger.error(err, logAttributes);
+    }
+  }
+
+  /*
+   * Closes the status timeline row that precedes the row we just created.
+   *
+   * The update is conditional instead of a blind updateOneById: it only touches
+   * the row if it starts at or before the new row (startsAt <= endsAt) and is
+   * either still open or currently closed at or after the new row's startsAt
+   * (endsAt >= :endsAt OR endsAt IS NULL). That keeps a concurrent or
+   * out-of-order writer from extending a row forward over a gap it does not own,
+   * and keeps a backfilled row from being closed at a time earlier than the row
+   * that actually follows it.
+   *
+   * startsAt uses <= and not <: on an exact startsAt tie (a backfill inserted at
+   * the same timestamp as the open predecessor) the predecessor must still be
+   * closed - at zero duration, which is harmless. With a strict < the tie would
+   * leave BOTH rows open forever: the reconciler deliberately never closes
+   * startsAt ties (its successor predicate is strictly later), so nothing would
+   * ever repair it, and the pair would read back as unbounded downtime.
+   *
+   * Note this closes only the single immediately-preceding row. If a monitor has
+   * more than one open row (the orphans this bug produced), the older ones are
+   * deliberately left alone: closing them here at the new row's startsAt would
+   * turn a near-zero-duration orphan into months of recorded downtime, which is
+   * worse than the current state and not repairable from the read path. Those
+   * rows must be closed at the startsAt of the row that actually follows each of
+   * them, which is the reconciler job's responsibility, not this write path's.
+   */
+  private async closePrecedingStatusTimeline(data: {
+    precedingStatusTimelineId: ObjectID;
+    endsAt: Date;
+    logAttributes: LogAttributes;
+  }): Promise<void> {
+    const updatedCount: number = await this.updateOneBy({
+      query: {
+        _id: data.precedingStatusTimelineId.toString(),
+        startsAt: QueryHelper.lessThanEqualTo(data.endsAt),
+        endsAt: QueryHelper.greaterThanEqualToOrNull(data.endsAt),
+      },
+      data: {
+        endsAt: data.endsAt,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (updatedCount === 0) {
+      /*
+       * The preceding row moved under us between onBeforeCreate reading it and
+       * this update - it was already closed at an earlier time, or it no longer
+       * starts before the new row. Leaving it as-is is correct; surface it so the
+       * reconciler has a signal to look at this monitor.
+       */
+      logger.warn(
+        `MonitorStatusTimeline: did not close preceding status timeline ${data.precedingStatusTimelineId.toString()} at ${data.endsAt.toString()}; it is no longer open and older than the new status.`,
+        data.logAttributes,
+      );
+    }
+  }
+
   @CaptureSpan()
   protected override async onCreateSuccess(
     onCreate: OnCreate<MonitorStatusTimeline>,
     createdItem: MonitorStatusTimeline,
   ): Promise<MonitorStatusTimeline> {
+    const logAttributes: LogAttributes = {
+      projectId: createdItem.projectId?.toString(),
+      monitorId: createdItem.monitorId?.toString(),
+    } as LogAttributes;
+
     if (!createdItem.monitorId) {
       throw new BadDataException("monitorId is null");
     }
@@ -202,34 +393,28 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
       throw new BadDataException("monitorStatusId is null");
     }
 
+    /*
+     * Everything below runs while the per-monitor mutex acquired in create() is
+     * still held, so the read of the predecessor in onBeforeCreate and the close
+     * of it here cannot interleave with another writer for the same monitor.
+     */
+
     // update the last status as ended.
 
-    logger.debug("Status Timeline Before this", {
-      projectId: createdItem.projectId?.toString(),
-      monitorId: createdItem.monitorId?.toString(),
-    } as LogAttributes);
-    logger.debug(onCreate.carryForward.statusTimelineBeforeThisStatus, {
-      projectId: createdItem.projectId?.toString(),
-      monitorId: createdItem.monitorId?.toString(),
-    } as LogAttributes);
+    logger.debug("Status Timeline Before this", logAttributes);
+    logger.debug(
+      onCreate.carryForward.statusTimelineBeforeThisStatus,
+      logAttributes,
+    );
 
-    logger.debug("Status Timeline After this", {
-      projectId: createdItem.projectId?.toString(),
-      monitorId: createdItem.monitorId?.toString(),
-    } as LogAttributes);
-    logger.debug(onCreate.carryForward.statusTimelineAfterThisStatus, {
-      projectId: createdItem.projectId?.toString(),
-      monitorId: createdItem.monitorId?.toString(),
-    } as LogAttributes);
+    logger.debug("Status Timeline After this", logAttributes);
+    logger.debug(
+      onCreate.carryForward.statusTimelineAfterThisStatus,
+      logAttributes,
+    );
 
-    logger.debug("Created Item", {
-      projectId: createdItem.projectId?.toString(),
-      monitorId: createdItem.monitorId?.toString(),
-    } as LogAttributes);
-    logger.debug(createdItem, {
-      projectId: createdItem.projectId?.toString(),
-      monitorId: createdItem.monitorId?.toString(),
-    } as LogAttributes);
+    logger.debug("Created Item", logAttributes);
+    logger.debug(createdItem, logAttributes);
 
     /*
      * now there are three cases.
@@ -237,57 +422,49 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
      */
     if (!onCreate.carryForward.statusTimelineBeforeThisStatus) {
       // This is the first status, no need to update previous status.
-      logger.debug("This is the first status.", {
-        projectId: createdItem.projectId?.toString(),
-        monitorId: createdItem.monitorId?.toString(),
-      } as LogAttributes);
+      logger.debug("This is the first status.", logAttributes);
     } else if (!onCreate.carryForward.statusTimelineAfterThisStatus) {
       /*
        * 2. This is the last status.
        * Update the previous status to end at the start of this status.
        */
-      await this.updateOneById({
-        id: onCreate.carryForward.statusTimelineBeforeThisStatus.id!,
-        data: {
-          endsAt: createdItem.startsAt!,
-        },
-        props: {
-          isRoot: true,
-        },
+      await this.closePrecedingStatusTimeline({
+        precedingStatusTimelineId:
+          onCreate.carryForward.statusTimelineBeforeThisStatus.id!,
+        endsAt: createdItem.startsAt!,
+        logAttributes: logAttributes,
       });
-      logger.debug("This is the last status.", {
-        projectId: createdItem.projectId?.toString(),
-        monitorId: createdItem.monitorId?.toString(),
-      } as LogAttributes);
+      logger.debug("This is the last status.", logAttributes);
     } else {
       /*
        * 3. This is in the middle.
        * Update the previous status to end at the start of this status.
        */
-      await this.updateOneById({
-        id: onCreate.carryForward.statusTimelineBeforeThisStatus.id!,
-        data: {
-          endsAt: createdItem.startsAt!,
-        },
-        props: {
-          isRoot: true,
-        },
+      await this.closePrecedingStatusTimeline({
+        precedingStatusTimelineId:
+          onCreate.carryForward.statusTimelineBeforeThisStatus.id!,
+        endsAt: createdItem.startsAt!,
+        logAttributes: logAttributes,
       });
 
-      // Update the next status to start at the end of this status.
-      await this.updateOneById({
-        id: onCreate.carryForward.statusTimelineAfterThisStatus.id!,
-        data: {
-          startsAt: createdItem.endsAt!,
-        },
-        props: {
-          isRoot: true,
-        },
-      });
-      logger.debug("This status is in the middle.", {
-        projectId: createdItem.projectId?.toString(),
-        monitorId: createdItem.monitorId?.toString(),
-      } as LogAttributes);
+      /*
+       * Update the next status to start at the end of this status. endsAt was
+       * set to the next row's startsAt in onBeforeCreate, so this is normally a
+       * no-op; it is guarded because writing an empty startsAt onto the next row
+       * would corrupt it.
+       */
+      if (createdItem.endsAt) {
+        await this.updateOneById({
+          id: onCreate.carryForward.statusTimelineAfterThisStatus.id!,
+          data: {
+            startsAt: createdItem.endsAt,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+      logger.debug("This status is in the middle.", logAttributes);
     }
 
     if (!createdItem.endsAt) {
@@ -303,10 +480,22 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
         props: onCreate.createBy.props,
       });
     }
+    return createdItem;
+  }
 
-    if (onCreate.carryForward.mutex) {
-      const mutex: SemaphoreMutex = onCreate.carryForward.mutex;
-      await Semaphore.release(mutex);
+  /*
+   * Writes the monitor feed item (and its workspace notification, which can be
+   * third-party HTTP to Slack/Teams) for a status change. Called from create()
+   * AFTER the per-monitor mutex has been released - see the comment there. Kept
+   * out of onCreateSuccess on purpose: everything in onCreateSuccess runs while
+   * the mutex is held.
+   */
+  private async createStatusChangeFeedItem(
+    createdItem: MonitorStatusTimeline,
+    createBy: CreateBy<MonitorStatusTimeline>,
+  ): Promise<void> {
+    if (!createdItem.monitorId || !createdItem.monitorStatusId) {
+      return;
     }
 
     const monitorStatus: MonitorStatus | null =
@@ -354,19 +543,15 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
         ` Changed Monitor **[${monitorName}](${(await MonitorService.getMonitorLinkInDashboard(projectId!, monitorId!)).toString()}) State** to **` +
         stateName +
         "**",
-      moreInformationInMarkdown: `**Cause:** 
+      moreInformationInMarkdown: `**Cause:**
     ${createdItem.rootCause}`,
-      userId: createdItem.createdByUserId || onCreate.createBy.props.userId,
+      userId: createdItem.createdByUserId || createBy.props.userId,
       workspaceNotification: {
         sendWorkspaceNotification: true,
         notifyUserId:
-          createdItem.createdByUserId ||
-          onCreate.createBy.props.userId ||
-          undefined,
+          createdItem.createdByUserId || createBy.props.userId || undefined,
       },
     });
-
-    return createdItem;
   }
 
   @CaptureSpan()

package/Server/Services/StatusPageService.ts

@@ -65,7 +65,7 @@ import IncidentService from "./IncidentService";
 import MonitorStatusTimeline from "../../Models/DatabaseModels/MonitorStatusTimeline";
 import MonitorStatusTimelineService from "./MonitorStatusTimelineService";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
-import UptimeUtil from "../../Utils/Uptime/UptimeUtil";
+import UptimeUtil, { UptimeWindow } from "../../Utils/Uptime/UptimeUtil";
 import UptimePrecision from "../../Types/StatusPage/UptimePrecision";
 import IP from "../../Types/IP/IP";
 import NotAuthenticatedException from "../../Types/Exception/NotAuthenticatedException";
@@ -708,24 +708,51 @@ export class Service extends DatabaseService<StatusPage> {
     let monitorStatusTimelines: Array<MonitorStatusTimeline> = [];
 
     if (data.monitorIds.length > 0) {
+      /*
+       * Select every row that actually OVERLAPS [startDate, endDate]:
+       * it started on or before the window ends, and it either ended on or after the window
+       * started or is still open (endsAt IS NULL).
+       *
+       * This used to be two queries that were concatenated - one keyed on
+       * `endsAt BETWEEN startDate AND endDate`, and one on `endsAt IS NULL` with NO date bound
+       * at all. That had two bugs. The unbounded open-row query pulled orphaned rows from
+       * months (or years) before the window into the report, and the endsAt-only window
+       * excluded any event that spanned the ENTIRE window but closed after it, so a total
+       * outage could render as a silent 100% uptime.
+       *
+       * `greaterThanEqualToOrNull` emits `(endsAt >= :date or endsAt IS NULL)`. Note that
+       * `greaterThanOrNull` emits exactly the same `>=` SQL despite its name, so the explicit
+       * name is used here to keep the intent readable.
+       *
+       * Supported by the (monitorId, startsAt) and (endsAt) indexes.
+       */
       monitorStatusTimelines = await MonitorStatusTimelineService.findBy({
         query: {
           monitorId: QueryHelper.any(data.monitorIds),
-          endsAt: QueryHelper.inBetween(startDate, endDate),
+          startsAt: QueryHelper.lessThanEqualTo(endDate),
+          endsAt: QueryHelper.greaterThanEqualToOrNull(startDate),
         },
         select: {
           monitorId: true,
           createdAt: true,
           endsAt: true,
           startsAt: true,
+          /*
+           * `_id` is selected explicitly because UptimeUtil reads `monitorStatus.id` (a getter
+           * over `_id`) to match events against the downtime statuses. It works today only
+           * because SelectUtil.sanitizeSelect force injects `_id: true` into every relation
+           * select - state that dependency here rather than rely on it implicitly. Note `_id`
+           * is the decorated primary column; `id` itself has no TableColumnMetadata.
+           */
           monitorStatus: {
+            _id: true,
             name: true,
             color: true,
             priority: true,
           } as any,
         },
         sort: {
-          createdAt: SortOrder.Descending,
+          startsAt: SortOrder.Descending,
         },
         skip: 0,
         limit: LIMIT_MAX, // This can be optimized.
@@ -734,42 +761,19 @@ export class Service extends DatabaseService<StatusPage> {
         },
       });
 
-      monitorStatusTimelines = monitorStatusTimelines.concat(
-        await MonitorStatusTimelineService.findBy({
-          query: {
-            monitorId: QueryHelper.any(data.monitorIds),
-            endsAt: QueryHelper.isNull(),
-          },
-          select: {
-            monitorId: true,
-            createdAt: true,
-            endsAt: true,
-            startsAt: true,
-            monitorStatus: {
-              name: true,
-              color: true,
-              priority: true,
-            } as any,
-          },
-          sort: {
-            createdAt: SortOrder.Descending,
-          },
-          skip: 0,
-          limit: LIMIT_MAX, // This can be optimized.
-          props: {
-            isRoot: true,
-          },
-        }),
-      );
-
-      // sort monitorStatusTimelines by createdAt.
+      /*
+       * Sort by startsAt and NOT by createdAt. createdAt is generated DB side with now(),
+       * while startsAt is generated on the worker pod with moment(). Those are different
+       * clocks with measured skew, so ordering by createdAt does not reliably reproduce the
+       * real chronological order of the timeline.
+       */
       monitorStatusTimelines = monitorStatusTimelines.sort(
         (a: MonitorStatusTimeline, b: MonitorStatusTimeline) => {
-          if (!a.createdAt || !b.createdAt) {
+          if (!a.startsAt || !b.startsAt) {
             return 0;
           }
 
-          return b.createdAt!.getTime() - a.createdAt!.getTime();
+          return b.startsAt!.getTime() - a.startsAt!.getTime();
         },
       );
     }
@@ -1201,6 +1205,18 @@ export class Service extends DatabaseService<StatusPage> {
     const startDate: Date = OneUptimeDate.getSomeDaysAgo(numberOfDays);
     const startAndEndDate: string = `${numberOfDays} days (${OneUptimeDate.getDateAsUserFriendlyLocalFormattedString(startDate, true)} - ${OneUptimeDate.getDateAsUserFriendlyLocalFormattedString(endDate, true)})`;
 
+    /*
+     * The report is explicitly "the last N days", so this window is what every uptime number
+     * below has to be measured against. It used to be computed only to fetch the timeline and
+     * was then discarded, which let an event that started before the window contribute its
+     * whole duration to the downtime total, and made the denominator "first event -> now"
+     * instead of the window - so the reported percentage drifted upwards every day.
+     */
+    const reportWindow: UptimeWindow = {
+      startDate: startDate,
+      endDate: endDate,
+    };
+
     if (statusPageResources.length === 0) {
       return {
         reportDates: startAndEndDate,
@@ -1260,6 +1276,7 @@ export class Service extends DatabaseService<StatusPage> {
         timelineForThisResource,
         resource.uptimePercentPrecision || UptimePrecision.TWO_DECIMAL,
         statusPage.downtimeMonitorStatuses!,
+        reportWindow,
       );
       const downtime: {
         totalDowntimeInSeconds: number;
@@ -1267,6 +1284,7 @@ export class Service extends DatabaseService<StatusPage> {
       } = UptimeUtil.getTotalDowntimeInSeconds(
         timelineForThisResource,
         statusPage.downtimeMonitorStatuses!,
+        reportWindow,
       );
 
       const reportItem: StatusPageReportItem = {
@@ -1299,6 +1317,7 @@ export class Service extends DatabaseService<StatusPage> {
     } = UptimeUtil.getTotalDowntimeInSeconds(
       timeline,
       statusPage.downtimeMonitorStatuses!,
+      reportWindow,
     );
 
     return {
@@ -1462,6 +1481,12 @@ export class Service extends DatabaseService<StatusPage> {
           },
           monitorGroupId: true,
           order: true,
+          /*
+           * needed so the emailed report rounds to the same precision the status page renders
+           * with. Without it `resource.uptimePercentPrecision` is always undefined and the
+           * report silently falls back to two decimals.
+           */
+          uptimePercentPrecision: true,
         },
         skip: 0,
         limit: LIMIT_PER_PROJECT,