@oneuptime/common 11.7.4 → 12.0.0

package/Server/Services/RunnerService.ts

@@ -0,0 +1,172 @@
+import CreateBy from "../Types/Database/CreateBy";
+import { OnCreate } from "../Types/Database/Hooks";
+import DatabaseService from "./DatabaseService";
+import ObjectID from "../../Types/ObjectID";
+import Version from "../../Types/Version";
+import BadDataException from "../../Types/Exception/BadDataException";
+import Model, {
+  RunnerConnectionStatus,
+} from "../../Models/DatabaseModels/Runner";
+import OneUptimeDate from "../../Types/Date";
+import { JSONObject } from "../../Types/JSON";
+import QueryHelper from "../Types/Database/QueryHelper";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+
+/*
+ * How recently a Runner must have heartbeated to count as online. Matches the
+ * window AIAgentService.isAgentAlive uses, and comfortably clears the Runner's
+ * 60s heartbeat interval.
+ */
+const RUNNER_ALIVE_WINDOW_IN_MINUTES: number = 5;
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    if (!createBy.data.key) {
+      createBy.data.key = ObjectID.generate().toString();
+    }
+
+    if (!createBy.data.agentVersion) {
+      createBy.data.agentVersion = new Version("1.0.0");
+    }
+
+    if (!createBy.data.connectionStatus) {
+      createBy.data.connectionStatus = RunnerConnectionStatus.Disconnected;
+    }
+
+    return { createBy, carryForward: [] };
+  }
+
+  @CaptureSpan()
+  public async findByIdAndKey(data: {
+    agentId: ObjectID;
+    agentKey: string;
+  }): Promise<Model | null> {
+    if (!data.agentId || !data.agentKey) {
+      return null;
+    }
+
+    return this.findOneBy({
+      query: {
+        _id: data.agentId.toString(),
+        key: data.agentKey,
+      },
+      select: {
+        _id: true,
+        projectId: true,
+        name: true,
+        canRunRunbooks: true,
+        canRunCodeFixTasks: true,
+        canRunAiCommands: true,
+      },
+      props: { isRoot: true },
+    });
+  }
+
+  /*
+   * Runners in this project that have opted into AI-composed commands and
+   * heartbeated recently. Same liveness semantics as the code-fix lookup:
+   * lastAlive as a query predicate, never a sort over possibly-NULL rows.
+   */
+  @CaptureSpan()
+  public async getOnlineAiCommandRunnersForProject(data: {
+    projectId: ObjectID;
+    limit?: number | undefined;
+  }): Promise<Array<Model>> {
+    return this.findBy({
+      query: {
+        projectId: data.projectId,
+        canRunAiCommands: true,
+        lastAlive: QueryHelper.greaterThan(
+          OneUptimeDate.getSomeMinutesAgo(RUNNER_ALIVE_WINDOW_IN_MINUTES),
+        ),
+      },
+      select: {
+        _id: true,
+        name: true,
+        description: true,
+        lastAlive: true,
+        hostInfo: true,
+      },
+      limit: data.limit ?? 25,
+      skip: 0,
+      props: { isRoot: true },
+    });
+  }
+
+  /*
+   * A Runner in this project that is allowed to take code-fix work and has
+   * heartbeated recently. Liveness is judged on lastAlive alone: unlike
+   * AIAgent, nothing ever flips a Runner's connectionStatus back to
+   * Disconnected, so a row that once connected would look Connected forever.
+   */
+  @CaptureSpan()
+  public async getOnlineCodeFixRunnerForProject(
+    projectId: ObjectID,
+  ): Promise<Model | null> {
+    /*
+     * The recency window is a QUERY predicate, not a post-filter on a sorted
+     * row: `ORDER BY "lastAlive" DESC` in Postgres puts NULLs FIRST, so a
+     * Runner row that was created but never started would be picked ahead of
+     * one that is heartbeating right now, and the project would read as
+     * having no agent.
+     */
+    const runners: Array<Model> = await this.findBy({
+      query: {
+        projectId: projectId,
+        canRunCodeFixTasks: true,
+        lastAlive: QueryHelper.greaterThan(
+          OneUptimeDate.getSomeMinutesAgo(RUNNER_ALIVE_WINDOW_IN_MINUTES),
+        ),
+      },
+      select: {
+        _id: true,
+        name: true,
+        lastAlive: true,
+      },
+      limit: 1,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    return runners[0] || null;
+  }
+
+  @CaptureSpan()
+  public async heartbeat(data: {
+    agentId: ObjectID;
+    agentVersion?: Version | undefined;
+    hostInfo?: JSONObject | undefined;
+  }): Promise<void> {
+    if (!data.agentId) {
+      throw new BadDataException("agentId is required");
+    }
+
+    const update: JSONObject = {
+      lastAlive: OneUptimeDate.getCurrentDate(),
+      connectionStatus: RunnerConnectionStatus.Connected,
+    };
+
+    if (data.agentVersion) {
+      update["agentVersion"] = data.agentVersion;
+    }
+
+    if (data.hostInfo) {
+      update["hostInfo"] = data.hostInfo;
+    }
+
+    await this.updateOneById({
+      id: data.agentId,
+      data: update as never,
+      props: { isRoot: true },
+    });
+  }
+}
+
+export default new Service();

package/Server/Services/ScheduledMaintenanceStateService.ts

@@ -1,3 +1,4 @@
+import logger from "../Utils/Logger";
 import CreateBy from "../Types/Database/CreateBy";
 import DeleteBy from "../Types/Database/DeleteBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
@@ -153,17 +154,30 @@ export class Service extends DatabaseService<ScheduledMaintenanceState> {
         newOrder = scheduledMaintenanceState.order! - 1;
       }
 
-      await this.updateOneBy({
-        query: {
-          _id: scheduledMaintenanceState._id!,
-        },
-        data: {
-          order: newOrder,
-        },
-        props: {
-          isRoot: true,
-        },
-      });
+      /*
+       * Concurrent deletes (e.g. Terraform destroying several items in
+       * parallel) can soft-delete a row between the findBy above and
+       * this update; save() would then INSERT with a null projectId and
+       * fail the whole delete with a 500. A row that vanished
+       * mid-rearrange needs no repositioning - skip it.
+       */
+      try {
+        await this.updateOneBy({
+          query: {
+            _id: scheduledMaintenanceState._id!,
+          },
+          data: {
+            order: newOrder,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      } catch (err) {
+        logger.warn(
+          `rearrange: skipping row (likely deleted concurrently): ${err}`,
+        );
+      }
     }
   }
 

package/Server/Services/ScheduledMaintenanceStateTimelineService.ts

@@ -646,13 +646,27 @@ export class Service extends DatabaseService<ScheduledMaintenanceStateTimeline>
       scheduledMaintenanceEvent.monitors &&
       scheduledMaintenanceEvent.monitors.length > 0
     ) {
-      // get resolved monitor state.
+      /*
+       * Resolve monitors back to the project's operational status once
+       * maintenance ends. A project can hold more than one operational state,
+       * so this lookup MUST be deterministic: without an explicit sort
+       * findOneBy falls back to `createdAt DESC` and would resolve monitors
+       * into whichever operational status was created most recently rather
+       * than the seeded default. Order by priority ascending (seeded default
+       * operational status is priority 0), tie-broken by the oldest row,
+       * matching MonitorService.onBeforeCreate so a monitor's operational
+       * status stays the same canonical one across its lifecycle.
+       */
       const resolvedMonitorState: MonitorStatus | null =
         await MonitorStatusService.findOneBy({
           query: {
             projectId: scheduledMaintenanceEvent.projectId!,
             isOperationalState: true,
           },
+          sort: {
+            priority: SortOrder.Ascending,
+            createdAt: SortOrder.Ascending,
+          },
           props: {
             isRoot: true,
           },

package/Server/Services/ServiceService.ts

@@ -34,8 +34,14 @@ export class Service extends DatabaseService<Model> {
   protected override async onBeforeCreate(
     createBy: CreateBy<Model>,
   ): Promise<OnCreate<Model>> {
-    // select a random color.
-    createBy.data.serviceColor = ArrayUtil.selectItemByRandom(BrightColors);
+    /*
+     * Select a random color when the caller did not provide one. API
+     * clients (e.g. Terraform) may set an explicit color; overwriting it
+     * made the field impossible to manage declaratively.
+     */
+    if (!createBy.data.serviceColor) {
+      createBy.data.serviceColor = ArrayUtil.selectItemByRandom(BrightColors);
+    }
 
     return {
       carryForward: null,

package/Server/Services/StatusPageService.ts

@@ -75,23 +75,27 @@ import {
   MASTER_PASSWORD_COOKIE_IDENTIFIER,
   MASTER_PASSWORD_REQUIRED_MESSAGE,
 } from "../../Types/StatusPage/MasterPassword";
-
-export interface StatusPageReportItem {
-  resourceName: string;
-  totalIncidentCount: number;
-  uptimePercent: number;
-  uptimePercentAsString: string;
-  downtimeInHoursAndMinutes: string;
-}
-
-export interface StatusPageReport {
-  reportDates: string; // start date and end date in string. e.g. "01 July 2021 - 14 July 2021"
-  totalResources: number;
-  totalIncidents: number;
-  averageUptimePercent: string;
-  totalDowntimeInHoursAndMinutes: string;
-  resources: Array<StatusPageReportItem>;
-}
+import StatusPageGroup from "../../Models/DatabaseModels/StatusPageGroup";
+import StatusPageGroupService from "./StatusPageGroupService";
+import StatusPageGroupTreeUtil from "../../Utils/StatusPage/GroupTree";
+import StatusPageReportTreeUtil, {
+  StatusPageReportResourceEntry,
+  StatusPageReportStructure,
+} from "../../Utils/StatusPage/Report";
+import {
+  StatusPageReport,
+  StatusPageReportGroup,
+  StatusPageReportGroupMetrics,
+  StatusPageReportItem,
+  StatusPageReportRow,
+} from "../../Types/StatusPage/StatusPageReport";
+
+export {
+  StatusPageReport,
+  StatusPageReportGroup,
+  StatusPageReportItem,
+  StatusPageReportRow,
+};
 
 export class Service extends DatabaseService<StatusPage> {
   /*
@@ -1262,6 +1266,10 @@ export class Service extends DatabaseService<StatusPage> {
         averageUptimePercent: "0%",
         totalDowntimeInHoursAndMinutes: "0",
         resources: [],
+        groups: [],
+        ungroupedResources: [],
+        rows: [],
+        hasGroups: false,
       };
     }
 
@@ -1284,7 +1292,13 @@ export class Service extends DatabaseService<StatusPage> {
         endDate: endDate,
       });
 
-    const reportItems: Array<StatusPageReportItem> = [];
+    const entries: Array<StatusPageReportResourceEntry> = [];
+
+    /*
+     * Kept alongside `entries` so a group can roll up the monitors of every
+     * resource beneath it without expanding monitor groups a second time.
+     */
+    const monitorIdsByResourceIndex: Array<Array<ObjectID>> = [];
 
     for (const resource of statusPageResources) {
       // for each of these resource, calculate uptime percent.
@@ -1324,27 +1338,30 @@ export class Service extends DatabaseService<StatusPage> {
         reportWindow,
       );
 
-      const reportItem: StatusPageReportItem = {
-        resourceName: resource.displayName || "",
-        totalIncidentCount: await this.getIncidentCountByMonitorIds({
-          monitorIds: monitorIdsForThisResource,
-          historyDays: data.historyDays,
-        }),
-        uptimePercent: uptimePercent,
-        uptimePercentAsString: `${uptimePercent}%`,
-        downtimeInHoursAndMinutes:
-          OneUptimeDate.convertMinutesToDaysHoursAndMinutes(
-            Math.ceil(downtime.totalDowntimeInSeconds / 60),
-          ),
-      };
+      entries.push({
+        statusPageResource: resource,
+        reportItem: {
+          resourceName: resource.displayName || "",
+          totalIncidentCount: await this.getIncidentCountByMonitorIds({
+            monitorIds: monitorIdsForThisResource,
+            historyDays: data.historyDays,
+          }),
+          uptimePercent: uptimePercent,
+          uptimePercentAsString: `${uptimePercent}%`,
+          downtimeInHoursAndMinutes:
+            OneUptimeDate.convertMinutesToDaysHoursAndMinutes(
+              Math.ceil(downtime.totalDowntimeInSeconds / 60),
+            ),
+        },
+      });
 
-      reportItems.push(reportItem);
+      monitorIdsByResourceIndex.push(monitorIdsForThisResource);
     }
 
     const avgUptimePercent: number =
-      reportItems.reduce((acc: number, item: StatusPageReportItem) => {
-        return acc + item.uptimePercent;
-      }, 0) / reportItems.length;
+      entries.reduce((acc: number, entry: StatusPageReportResourceEntry) => {
+        return acc + entry.reportItem.uptimePercent;
+      }, 0) / entries.length;
 
     const avgUptimePercentString: string = avgUptimePercent.toFixed(2) + "%";
 
@@ -1357,12 +1374,46 @@ export class Service extends DatabaseService<StatusPage> {
       reportWindow,
     );
 
+    /*
+     * The status page arranges its resources into a tree of groups and shows a
+     * rolled up number at every level. Recipients of this email usually have no
+     * login, so the report has to carry that hierarchy too - see
+     * StatusPageReportTreeUtil.
+     */
+    const statusPageGroups: Array<StatusPageGroup> =
+      await this.getStatusPageGroups({
+        statusPageId: data.statusPageId,
+      });
+
+    const groupMetricsByGroupId: Dictionary<StatusPageReportGroupMetrics> =
+      await this.getReportGroupMetrics({
+        statusPageGroups: statusPageGroups,
+        entries: entries,
+        monitorIdsByResourceIndex: monitorIdsByResourceIndex,
+        timeline: timeline,
+        downtimeMonitorStatuses: statusPage.downtimeMonitorStatuses || [],
+        reportWindow: reportWindow,
+        historyDays: data.historyDays,
+      });
+
+    const structure: StatusPageReportStructure = StatusPageReportTreeUtil.build(
+      {
+        entries: entries,
+        statusPageGroups: statusPageGroups,
+        groupMetricsByGroupId: groupMetricsByGroupId,
+      },
+    );
+
     return {
       reportDates: startAndEndDate,
       totalResources: statusPageResources.length,
       totalIncidents: incidentCount,
       averageUptimePercent: avgUptimePercentString,
-      resources: reportItems,
+      resources: structure.resources,
+      groups: structure.groups,
+      ungroupedResources: structure.resourcesWithoutGroup,
+      rows: structure.rows,
+      hasGroups: structure.groups.length > 0,
       totalDowntimeInHoursAndMinutes:
         OneUptimeDate.convertMinutesToDaysHoursAndMinutes(
           Math.ceil(totalDowntimeInSeconds.totalDowntimeInSeconds / 60),
@@ -1370,6 +1421,159 @@ export class Service extends DatabaseService<StatusPage> {
     };
   }
 
+  /*
+   * Uptime, downtime and incidents for each group, measured over the group and
+   * every group nested under it - the same set of resources the live status page
+   * rolls a group's number over.
+   */
+  @CaptureSpan()
+  public async getReportGroupMetrics(data: {
+    statusPageGroups: Array<StatusPageGroup>;
+    entries: Array<StatusPageReportResourceEntry>;
+    monitorIdsByResourceIndex: Array<Array<ObjectID>>;
+    timeline: Array<MonitorStatusTimeline>;
+    downtimeMonitorStatuses: Array<MonitorStatus>;
+    reportWindow: UptimeWindow;
+    historyDays: number;
+  }): Promise<Dictionary<StatusPageReportGroupMetrics>> {
+    const groupMetricsByGroupId: Dictionary<StatusPageReportGroupMetrics> = {};
+    const incidentCountByMonitorSet: Dictionary<number> = {};
+
+    for (const group of data.statusPageGroups) {
+      const groupId: string | undefined = group._id?.toString();
+
+      if (!groupId) {
+        continue;
+      }
+
+      const groupIdsInSubtree: Set<string> = new Set<string>(
+        StatusPageGroupTreeUtil.getGroupAndDescendants({
+          statusPageGroup: group,
+          statusPageGroups: data.statusPageGroups,
+        }).map((groupInSubtree: StatusPageGroup) => {
+          return groupInSubtree._id?.toString() || "";
+        }),
+      );
+
+      const uptimePercentsInSubtree: Array<number> = [];
+      const monitorIdsInSubtree: Dictionary<ObjectID> = {};
+
+      data.entries.forEach(
+        (entry: StatusPageReportResourceEntry, index: number) => {
+          const resourceGroupId: string | undefined =
+            entry.statusPageResource.statusPageGroupId?.toString();
+
+          if (!resourceGroupId || !groupIdsInSubtree.has(resourceGroupId)) {
+            return;
+          }
+
+          uptimePercentsInSubtree.push(entry.reportItem.uptimePercent);
+
+          for (const monitorId of data.monitorIdsByResourceIndex[index] || []) {
+            // de-duplicated: the same monitor can back more than one resource.
+            monitorIdsInSubtree[monitorId.toString()] = monitorId;
+          }
+        },
+      );
+
+      const monitorIds: Array<ObjectID> = Object.values(monitorIdsInSubtree);
+
+      const timelineForThisGroup: Array<MonitorStatusTimeline> =
+        data.timeline.filter((item: MonitorStatusTimeline) => {
+          return Boolean(
+            item.monitorId && monitorIdsInSubtree[item.monitorId.toString()],
+          );
+        });
+
+      const downtime: {
+        totalDowntimeInSeconds: number;
+        totalSecondsInTimePeriod: number;
+      } = UptimeUtil.getTotalDowntimeInSeconds(
+        timelineForThisGroup,
+        data.downtimeMonitorStatuses,
+        data.reportWindow,
+      );
+
+      /*
+       * Averaging the resources' percentages (rather than recomputing from the
+       * merged timeline) is what the live page does, so a group in the email
+       * shows the number a reader can compare against the page.
+       */
+      const uptimePercent: number =
+        uptimePercentsInSubtree.length > 0
+          ? UptimeUtil.calculateAvgUptimePercentage({
+              uptimePercentages: uptimePercentsInSubtree,
+              precision:
+                group.uptimePercentPrecision || UptimePrecision.TWO_DECIMAL,
+            })
+          : 0;
+
+      /*
+       * A chain like Corporate -> Region -> Market -> Unit rolls up the exact
+       * same monitors at every level, so the incident count is cached by the
+       * monitor set rather than issuing one identical query per level.
+       */
+      const monitorSetKey: string = monitorIds
+        .map((monitorId: ObjectID) => {
+          return monitorId.toString();
+        })
+        .sort()
+        .join(",");
+
+      let totalIncidentCount: number | undefined =
+        incidentCountByMonitorSet[monitorSetKey];
+
+      if (totalIncidentCount === undefined) {
+        totalIncidentCount =
+          monitorIds.length > 0
+            ? await this.getIncidentCountByMonitorIds({
+                monitorIds: monitorIds,
+                historyDays: data.historyDays,
+              })
+            : 0;
+
+        incidentCountByMonitorSet[monitorSetKey] = totalIncidentCount;
+      }
+
+      groupMetricsByGroupId[groupId] = {
+        uptimePercent: uptimePercent,
+        uptimePercentAsString: `${uptimePercent}%`,
+        downtimeInHoursAndMinutes:
+          OneUptimeDate.convertMinutesToDaysHoursAndMinutes(
+            Math.ceil(downtime.totalDowntimeInSeconds / 60),
+          ),
+        totalIncidentCount: totalIncidentCount,
+      };
+    }
+
+    return groupMetricsByGroupId;
+  }
+
+  @CaptureSpan()
+  public async getStatusPageGroups(data: {
+    statusPageId: ObjectID;
+  }): Promise<Array<StatusPageGroup>> {
+    return await StatusPageGroupService.findBy({
+      query: {
+        statusPageId: data.statusPageId,
+      },
+      select: {
+        name: true,
+        order: true,
+        parentStatusPageGroupId: true,
+        uptimePercentPrecision: true,
+      },
+      sort: {
+        order: SortOrder.Ascending,
+      },
+      skip: 0,
+      limit: LIMIT_PER_PROJECT,
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+
   @CaptureSpan()
   public async getIncidentCountByMonitorIds(data: {
     monitorIds: Array<ObjectID>;

package/Server/Services/StatusPageSubscriberNotificationTemplateService.ts

@@ -277,7 +277,26 @@ export class Service extends DatabaseService<Model> {
           {
             name: "report.resources",
             description:
-              "Array of per-resource rows (resourceName, uptimePercentAsString, downtimeInHoursAndMinutes, totalIncidentCount) to loop over with {{#each}}",
+              "Array of per-resource rows (resourceName, uptimePercentAsString, downtimeInHoursAndMinutes, totalIncidentCount, groupName, groupPath) to loop over with {{#each}}",
+          },
+          {
+            name: "report.hasGroups",
+            description:
+              "true when the status page organises its resources into groups",
+          },
+          {
+            name: "report.rows",
+            description:
+              "The status page's group hierarchy flattened into render order (isGroup, name, depth, indentInPixels, uptimePercentAsString, downtimeInHoursAndMinutes, totalIncidentCount, totalResources) to loop over with {{#each}}",
+          },
+          {
+            name: "report.groups",
+            description:
+              "The group hierarchy as a nested tree (groupName, groupPath, depth, uptimePercentAsString, downtimeInHoursAndMinutes, totalIncidentCount, totalResources, resources, subGroups)",
+          },
+          {
+            name: "report.ungroupedResources",
+            description: "Per-resource rows for resources that are in no group",
           },
         ];
 

package/Server/Services/TelemetryExceptionService.ts

@@ -6,7 +6,7 @@ import ObjectID from "../../Types/ObjectID";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import BadDataException from "../../Types/Exception/BadDataException";
 import AIRunType from "../../Types/AI/AIRunType";
-import AIRunStatus from "../../Types/AI/AIRunStatus";
+import AIRunStatus, { AIRunStatusHelper } from "../../Types/AI/AIRunStatus";
 import CodeFixTaskType, {
   CodeFixTaskTypeHelper,
 } from "../../Types/AI/CodeFixTaskType";
@@ -433,14 +433,7 @@ export class Service extends DatabaseService<Model> {
           taskType === CodeFixTaskType.FixException
             ? QueryHelper.equalToOrNull(CodeFixTaskType.FixException)
             : taskType,
-        // Every terminal status — see AIRunStatusHelper.isTerminalStatus.
-        status: QueryHelper.notIn([
-          AIRunStatus.Completed,
-          AIRunStatus.NoFixFound,
-          AIRunStatus.Error,
-          AIRunStatus.Cancelled,
-          AIRunStatus.Stale,
-        ]),
+        status: QueryHelper.notIn(AIRunStatusHelper.terminalStatuses()),
       },
       select: {
         _id: true,
@@ -490,13 +483,7 @@ export class Service extends DatabaseService<Model> {
           data.taskType === CodeFixTaskType.FixException
             ? QueryHelper.equalToOrNull(CodeFixTaskType.FixException)
             : data.taskType,
-        status: QueryHelper.notIn([
-          AIRunStatus.Completed,
-          AIRunStatus.NoFixFound,
-          AIRunStatus.Error,
-          AIRunStatus.Cancelled,
-          AIRunStatus.Stale,
-        ]),
+        status: QueryHelper.notIn(AIRunStatusHelper.terminalStatuses()),
       },
       select: {
         triggeredByTelemetryExceptionId: true,