@oneuptime/common 11.5.11 → 11.5.12

package/Models/AnalyticsModels/NetworkFlow.ts

@@ -285,8 +285,19 @@ export default class NetworkFlow extends AnalyticsBaseModel {
        * (a retentionDate column computed at ingest, like Log/Metric) is
        * the phase-2 follow-up. Keyed on server-assigned ingestedAt so a
        * device with a wrong clock cannot make rows expire early.
+       *
+       * toDateTime() is NOT redundant: ingestedAt is DateTime64(9), so
+       * `ingestedAt + INTERVAL 30 DAY` stays DateTime64(9), and a TTL
+       * expression must evaluate to Date or DateTime. ClickHouse only began
+       * accepting DateTime64 there in 25.x; on 24.x it fails the CREATE with
+       * BAD_TTL_EXPRESSION, which aborts the whole boot schema-sync (it has
+       * no per-table error handling) and takes every table registered after
+       * this one down with it. The Helm chart ships `tag: latest` but tells
+       * operators to pin for production, so older servers are supported and
+       * this must stay version-portable. Narrowing to second precision is
+       * lossless for a 30-day retention window.
        */
-      ttlExpression: "ingestedAt + INTERVAL 30 DAY DELETE",
+      ttlExpression: "toDateTime(ingestedAt) + INTERVAL 30 DAY DELETE",
       defaultSortColumn: "flowStartAt",
     });
   }

package/Server/API/StatusPageAPI.ts

@@ -88,7 +88,7 @@ import StatusPageEventType from "../../Types/StatusPage/StatusPageEventType";
 import StatusPageResourceUptimeUtil from "../../Utils/StatusPage/ResourceUptime";
 import UptimePrecision from "../../Types/StatusPage/UptimePrecision";
 import { Green } from "../../Types/BrandColors";
-import UptimeUtil from "../../Utils/Uptime/UptimeUtil";
+import UptimeUtil, { UptimeWindow } from "../../Utils/Uptime/UptimeUtil";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import URL from "../../Types/API/URL";
 import SMS from "../../Types/SMS/SMS";
@@ -1265,6 +1265,17 @@ export default class StatusPageAPI extends BaseAPI<
           const downtimeMonitorStatuses: Array<MonitorStatus> =
             statusPage.downtimeMonitorStatuses || [];
 
+          /*
+           * this endpoint reports uptime over an explicit [startDate, endDate] range, so events
+           * have to be clipped to it and the denominator has to be the range itself. Without
+           * this an open (endsAt = null) row that started before the range contributes its
+           * entire duration to the downtime total.
+           */
+          const uptimeWindow: UptimeWindow = {
+            startDate: startDate,
+            endDate: endDate,
+          };
+
           type ResourceUptime = {
             statusPageResourceId: ObjectID;
             uptimePercent: number | null;
@@ -1369,6 +1380,7 @@ export default class StatusPageAPI extends BaseAPI<
                           resourceStatusTimelines,
                           precision,
                           downtimeMonitorStatuses,
+                          uptimeWindow,
                         );
 
                       resourceUptime.uptimePercent = uptimePercent;
@@ -1419,6 +1431,7 @@ export default class StatusPageAPI extends BaseAPI<
                           resourceStatusTimelines,
                           precision,
                           downtimeMonitorStatuses,
+                          uptimeWindow,
                         );
 
                       resourceUptime.uptimePercent = uptimePercent;

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -461,7 +461,6 @@ import { AddNetworkDeviceInventoryAndDiscoverySchedule1784211212164 } from "./17
 import { AddEnterpriseLicenseNotificationColumns1784218257664 } from "./1784218257664-AddEnterpriseLicenseNotificationColumns";
 import { AddAttributionColumnsToUserAndProject1784293516000 } from "./1784293516000-AddAttributionColumnsToUserAndProject";
 import { AddMarketingConversionTable1784298000000 } from "./1784298000000-AddMarketingConversionTable";
-import { MigrationName1784401962564 } from "./1784401962564-MigrationName";
 
 export default [
   InitialMigration,
@@ -927,5 +926,4 @@ export default [
   AddEnterpriseLicenseNotificationColumns1784218257664,
   AddAttributionColumnsToUserAndProject1784293516000,
   AddMarketingConversionTable1784298000000,
-  MigrationName1784401962564,
 ];

package/Server/Services/IncidentService.ts

@@ -15,7 +15,6 @@ import IncidentStateService from "./IncidentStateService";
 import IncidentStateTimelineService from "./IncidentStateTimelineService";
 import MonitorService from "./MonitorService";
 import MonitorStatusService from "./MonitorStatusService";
-import MonitorStatusTimelineService from "./MonitorStatusTimelineService";
 import OnCallDutyPolicyService from "./OnCallDutyPolicyService";
 import TeamMemberService from "./TeamMemberService";
 import UserService from "./UserService";
@@ -43,7 +42,6 @@ import IncidentState from "../../Models/DatabaseModels/IncidentState";
 import IncidentStateTimeline from "../../Models/DatabaseModels/IncidentStateTimeline";
 import Monitor from "../../Models/DatabaseModels/Monitor";
 import MonitorStatus from "../../Models/DatabaseModels/MonitorStatus";
-import MonitorStatusTimeline from "../../Models/DatabaseModels/MonitorStatusTimeline";
 import User from "../../Models/DatabaseModels/User";
 import { IsBillingEnabled } from "../EnvironmentConfig";
 import MutableMetricService from "./MutableMetricService";
@@ -2538,84 +2536,81 @@ ${incidentSeverity.name}
 
       if (resolvedMonitorState) {
         for (const monitor of monitors) {
-          //check state of the monitor.
-
-          const doesMonitorHasMoreActiveManualIncidents: boolean =
-            await this.doesMonitorHasMoreActiveManualIncidents(
-              monitor.id!,
-              projectId!,
-            );
-
-          if (doesMonitorHasMoreActiveManualIncidents) {
-            continue;
-          }
-
-          await MonitorService.updateOneById({
-            id: monitor.id!,
-            data: {
-              disableActiveMonitoringBecauseOfManualIncident: false,
-            },
-            props: {
-              isRoot: true,
-            },
-          });
-
           /*
-           * Don't flip the monitor to operational while other incidents
-           * are still open on it — e.g. a metric monitor with group-by
-           * may have one incident per series, and resolving one series
-           * shouldn't claim the whole monitor is healthy.
+           * Per-monitor isolation: one monitor failing here (for example the
+           * fail-closed status timeline lock, or a transient DB error) must not
+           * abort the loop and strand the REMAINING monitors with
+           * disableActiveMonitoringBecauseOfManualIncident still true - a
+           * monitor left in that state is skipped by probes and sits in its
+           * down status indefinitely, silently accruing downtime.
            */
-          const hasOtherActiveIncidents: boolean =
-            await this.doesMonitorHaveActiveIncidents(monitor.id!, projectId!);
+          try {
+            //check state of the monitor.
 
-          if (hasOtherActiveIncidents) {
-            continue;
-          }
+            const doesMonitorHasMoreActiveManualIncidents: boolean =
+              await this.doesMonitorHasMoreActiveManualIncidents(
+                monitor.id!,
+                projectId!,
+              );
 
-          const latestState: MonitorStatusTimeline | null =
-            await MonitorStatusTimelineService.findOneBy({
-              query: {
-                monitorId: monitor.id!,
-                projectId: projectId!,
-              },
-              select: {
-                _id: true,
-                monitorStatusId: true,
+            if (doesMonitorHasMoreActiveManualIncidents) {
+              continue;
+            }
+
+            await MonitorService.updateOneById({
+              id: monitor.id!,
+              data: {
+                disableActiveMonitoringBecauseOfManualIncident: false,
               },
               props: {
                 isRoot: true,
               },
-              sort: {
-                startsAt: SortOrder.Descending,
-              },
             });
 
-          if (
-            latestState &&
-            latestState.monitorStatusId?.toString() ===
-              resolvedMonitorState.id!.toString()
-          ) {
-            // already on this state. Skip.
-            continue;
-          }
+            /*
+             * Don't flip the monitor to operational while other incidents
+             * are still open on it — e.g. a metric monitor with group-by
+             * may have one incident per series, and resolving one series
+             * shouldn't claim the whole monitor is healthy.
+             */
+            const hasOtherActiveIncidents: boolean =
+              await this.doesMonitorHaveActiveIncidents(
+                monitor.id!,
+                projectId!,
+              );
 
-          const monitorStatusTimeline: MonitorStatusTimeline =
-            new MonitorStatusTimeline();
-          monitorStatusTimeline.monitorId = monitor.id!;
-          monitorStatusTimeline.projectId = projectId!;
-          monitorStatusTimeline.monitorStatusId = resolvedMonitorState.id!;
+            if (hasOtherActiveIncidents) {
+              continue;
+            }
 
-          if (startsAt) {
-            monitorStatusTimeline.startsAt = startsAt;
+            /*
+             * changeMonitorStatus performs the same latest-status dedupe check
+             * this loop used to do inline, and additionally absorbs the two
+             * recoverable error classes (status already set by a concurrent
+             * writer, lock not acquired after retries) by skipping the write,
+             * so the common failure modes never even reach the catch below.
+             * notifyOwners is true to preserve the previous behavior here: the
+             * created row had isOwnerNotified unset, so owners were notified.
+             */
+            await MonitorService.changeMonitorStatus(
+              projectId!,
+              [monitor.id!],
+              resolvedMonitorState.id!,
+              true, // notifyOwners - matches the pre-existing behavior of this loop.
+              undefined,
+              undefined,
+              {
+                isRoot: true,
+              },
+              startsAt,
+            );
+          } catch (err) {
+            logger.error(
+              `IncidentService.markMonitorsActiveForMonitoring: failed for monitor ${monitor.id?.toString()}; continuing with the remaining monitors.`,
+            );
+            logger.error(err);
+            continue;
           }
-
-          await MonitorStatusTimelineService.create({
-            data: monitorStatusTimeline,
-            props: {
-              isRoot: true,
-            },
-          });
         }
       }
     }

package/Server/Services/MonitorGroupService.ts

@@ -93,7 +93,16 @@ export class Service extends DatabaseService<MonitorGroup> {
               },
             ),
           ),
-          createdAt: QueryHelper.inBetween(startDate, endDate),
+          /*
+           * Select every row that OVERLAPS [startDate, endDate] - it started on or before the
+           * window ends, and it either ended on or after the window started or is still open
+           * (endsAt IS NULL). Filtering by createdAt-in-window missed rows that started before
+           * the window but carry downtime into it (e.g. a monitor Offline since before the
+           * window rendered as 100% uptime on the group page). Same predicate as
+           * StatusPageService.getMonitorStatusTimelineForStatusPage.
+           */
+          startsAt: QueryHelper.lessThanEqualTo(endDate),
+          endsAt: QueryHelper.greaterThanEqualToOrNull(startDate),
         },
         select: {
           createdAt: true,
@@ -108,7 +117,11 @@ export class Service extends DatabaseService<MonitorGroup> {
           } as any,
         },
         sort: {
-          createdAt: SortOrder.Ascending,
+          /*
+           * startsAt, not createdAt: they are different clocks (DB now() vs worker moment())
+           * with real skew, and startsAt is the order the timeline math relies on.
+           */
+          startsAt: SortOrder.Ascending,
         },
         skip: 0,
         limit: LIMIT_MAX, // This can be optimized.

package/Server/Services/MonitorService.ts

@@ -14,7 +14,12 @@ import MonitorOwnerTeamService from "./MonitorOwnerTeamService";
 import MonitorOwnerUserService from "./MonitorOwnerUserService";
 import MonitorProbeService from "./MonitorProbeService";
 import MonitorStatusService from "./MonitorStatusService";
-import MonitorStatusTimelineService from "./MonitorStatusTimelineService";
+import MonitorStatusTimelineService, {
+  MONITOR_STATUS_SAME_AS_PREVIOUS_ERROR_MESSAGE,
+  MONITOR_STATUS_TIMELINE_LOCK_ERROR_MESSAGE,
+} from "./MonitorStatusTimelineService";
+import ServerException from "../../Types/Exception/ServerException";
+import Sleep from "../../Types/Sleep";
 import ProbeService from "./ProbeService";
 import ProjectService, { CurrentPlan } from "./ProjectService";
 import TeamMemberService from "./TeamMemberService";
@@ -1733,13 +1738,96 @@ ${createdItem.description?.trim() || "No description provided."}
         statusTimeline.startsAt = startsAt;
       }
 
-      await MonitorStatusTimelineService.create({
-        data: statusTimeline,
+      await this.createStatusTimelineWithRetry({
+        statusTimeline: statusTimeline,
         props: props,
+        projectId: projectId,
+        monitorId: monitorId,
       });
     }
   }
 
+  /*
+   * Creates one status timeline row, absorbing the two error classes that are
+   * recoverable per monitor so one monitor cannot abort a caller's loop over
+   * many (incident resolve, scheduled maintenance end, incident create with
+   * changeMonitorStatusTo):
+   *
+   *   - "same as previous" from onBeforeCreate's dedupe check: a concurrent
+   *     writer (or an earlier backfilled row at this startsAt) already put the
+   *     monitor in this status, so the desired state holds. Idempotent no-op.
+   *
+   *   - the fail-closed lock error from MonitorStatusTimelineService.create():
+   *     retried a few times with a short delay, because the common causes (a
+   *     transient Redis blip, or a concurrent writer holding the per-monitor
+   *     mutex) clear quickly. This retry is also what protects the INITIAL
+   *     status row written from MonitorService.onCreateSuccess - without it a
+   *     lock failure at monitor-create time leaves a Manual monitor with no
+   *     timeline forever, since Manual monitors are never probed and so never
+   *     self-heal. If all attempts fail, log and continue: for probed monitors
+   *     the next probe result recreates the transition, and failing the caller
+   *     outright would strand its remaining monitors instead.
+   *
+   * Every other error still propagates.
+   */
+  private async createStatusTimelineWithRetry(data: {
+    statusTimeline: MonitorStatusTimeline;
+    props: DatabaseCommonInteractionProps;
+    projectId: ObjectID;
+    monitorId: ObjectID;
+  }): Promise<void> {
+    const maxAttempts: number = 3;
+    const retryDelayInMs: number = 2000;
+
+    const logAttributes: LogAttributes = {
+      projectId: data.projectId.toString(),
+      monitorId: data.monitorId.toString(),
+    } as LogAttributes;
+
+    for (let attempt: number = 1; attempt <= maxAttempts; attempt++) {
+      try {
+        await MonitorStatusTimelineService.create({
+          data: data.statusTimeline,
+          props: data.props,
+        });
+        return;
+      } catch (err) {
+        if (
+          err instanceof BadDataException &&
+          err.message === MONITOR_STATUS_SAME_AS_PREVIOUS_ERROR_MESSAGE
+        ) {
+          logger.debug(
+            `changeMonitorStatus: monitor ${data.monitorId.toString()} is already in the requested status; skipping duplicate status timeline.`,
+          );
+          return;
+        }
+
+        const isLockError: boolean =
+          err instanceof ServerException &&
+          err.message === MONITOR_STATUS_TIMELINE_LOCK_ERROR_MESSAGE;
+
+        if (isLockError && attempt < maxAttempts) {
+          logger.warn(
+            `changeMonitorStatus: could not acquire the status timeline lock for monitor ${data.monitorId.toString()} (attempt ${attempt} of ${maxAttempts}); retrying.`,
+            logAttributes,
+          );
+          await Sleep.sleep(retryDelayInMs);
+          continue;
+        }
+
+        if (isLockError) {
+          logger.error(
+            `changeMonitorStatus: could not acquire the status timeline lock for monitor ${data.monitorId.toString()} after ${maxAttempts} attempt(s); skipping this status change. The monitor keeps its current status.`,
+            logAttributes,
+          );
+          return;
+        }
+
+        throw err;
+      }
+    }
+  }
+
   @CaptureSpan()
   public async getWorkspaceChannelForMonitor(data: {
     monitorId: ObjectID;