@oneuptime/common 14.0.6 → 14.0.8

package/Server/Utils/StatusPage/SubscriberNotificationTiming.ts

@@ -0,0 +1,133 @@
+import LIMIT_MAX from "../../../Types/Database/LimitMax";
+import OneUptimeDate from "../../../Types/Date";
+
+/*
+ * How long the incident and episode subscriber jobs may spend sending, and
+ * how long a notification may sit In progress before it is taken to have been
+ * interrupted.
+ *
+ * A job sends each notification - an incident created, a state change, a
+ * public note, a postmortem, or an episode's - by awaiting every message at a
+ * bounded concurrency, so a send takes as long as its slowest subscribers
+ * rather than returning the moment the messages are queued. These numbers
+ * keep that bounded, and tie the job's timeout, a notification's send window
+ * and the sweeper's threshold together so they cannot drift apart:
+ *
+ * - SEND_CONCURRENCY messages are in flight at once, per notification.
+ * - SUBSCRIBERS_PER_READ subscribers are read from a status page at a time:
+ *   LIMIT_MAX, the cap one read used to stop at silently. A page with more is
+ *   read in batches (SubscriberNotificationFanOut).
+ * - A message still unanswered after SEND_TIMEOUT_IN_MS is counted failed and
+ *   its slot freed. That is longer than any sender's own retries can take -
+ *   the Notification service tries SMTP three times with a 60 second connect
+ *   timeout each (about 3 minutes 5 seconds), and the Slack and webhook
+ *   senders retry three times with 2, 4 and 8 second backoffs - so it only
+ *   cuts off a request that hangs. The request itself is not cancelled.
+ * - A notification starts no new message once SEND_WINDOW_IN_MS has passed
+ *   since it was claimed. What it has not reached is recorded as not sent,
+ *   and the notification settles as Failed.
+ *
+ *   The math, for one full read of a status page: 10,000 subscribers at 20 in
+ *   flight is 500 rounds, and 20 minutes over 500 rounds allows 2.4 seconds a
+ *   message on average. An email, SMS or webhook through the Notification
+ *   service normally answers in well under a second, so a 10,000-subscriber
+ *   page fits with room to spare, and at half a second a message one
+ *   notification reaches about 48,000 subscribers in its window.
+ *
+ * - One notification can therefore take at most NOTIFICATION_MAX_IN_MS: its
+ *   send window, plus the timeout of the messages still in flight when the
+ *   window closes, plus a minute for the feed item and the status write.
+ * - JOB_TIMEOUT_IN_MS is each job's RunCron timeoutInMS. A run claims a new
+ *   notification only while that notification could still finish inside the
+ *   job's timeout (LATEST_CLAIM_IN_MS into the run), so a run never outlives
+ *   its timeout: a timed-out queue job is only stopped being waited for,
+ *   never cancelled, and would otherwise carry on sending untracked. The
+ *   notifications it leaves Pending are claimed by the runs that follow, one
+ *   a minute, which can run alongside it: a claim is atomic
+ *   (SubscriberNotificationClaim), so each notification is sent once. How
+ *   many runs of a job may be sending at once, across every worker, is
+ *   bounded too (SubscriberNotificationRunLimit), so slow sends cannot fill
+ *   the Worker queue every other cron shares.
+ * - STUCK_AFTER_IN_MS is when the sweeper
+ *   (StatusPageSubscriber:TimeoutStuckNotifications) takes a notification
+ *   still In progress to have been interrupted - its worker crashed, was
+ *   redeployed or lost its database connection - and marks it Failed. A
+ *   running send is never In progress that long, so the margin over the job
+ *   timeout only has to cover clock skew between the worker and the database.
+ */
+export default class SubscriberNotificationTiming {
+  public static readonly SEND_CONCURRENCY: number = 20;
+
+  public static readonly SUBSCRIBERS_PER_READ: number = LIMIT_MAX;
+
+  public static readonly SEND_TIMEOUT_IN_MS: number =
+    OneUptimeDate.convertMinutesToMilliseconds(4);
+
+  public static readonly SEND_WINDOW_IN_MS: number =
+    OneUptimeDate.convertMinutesToMilliseconds(20);
+
+  public static readonly SETTLE_ALLOWANCE_IN_MS: number =
+    OneUptimeDate.convertMinutesToMilliseconds(1);
+
+  public static readonly NOTIFICATION_MAX_IN_MS: number =
+    SubscriberNotificationTiming.SEND_WINDOW_IN_MS +
+    SubscriberNotificationTiming.SEND_TIMEOUT_IN_MS +
+    SubscriberNotificationTiming.SETTLE_ALLOWANCE_IN_MS;
+
+  public static readonly JOB_TIMEOUT_IN_MS: number =
+    OneUptimeDate.convertMinutesToMilliseconds(30);
+
+  public static readonly LATEST_CLAIM_IN_MS: number =
+    SubscriberNotificationTiming.JOB_TIMEOUT_IN_MS -
+    SubscriberNotificationTiming.NOTIFICATION_MAX_IN_MS;
+
+  public static readonly STUCK_AFTER_IN_MS: number =
+    SubscriberNotificationTiming.JOB_TIMEOUT_IN_MS +
+    OneUptimeDate.convertMinutesToMilliseconds(10);
+
+  // The clock of one job run, started when the run starts.
+  public static startRun(now?: number): SubscriberNotificationRunClock {
+    return new SubscriberNotificationRunClock(now ?? Date.now());
+  }
+
+  // The send window of one notification, started when it is claimed.
+  public static startSendWindow(
+    now?: number,
+  ): SubscriberNotificationSendWindow {
+    return new SubscriberNotificationSendWindow(
+      (now ?? Date.now()) + SubscriberNotificationTiming.SEND_WINDOW_IN_MS,
+    );
+  }
+}
+
+export class SubscriberNotificationRunClock {
+  private readonly startedAt: number;
+
+  public constructor(startedAt: number) {
+    this.startedAt = startedAt;
+  }
+
+  /*
+   * Whether this run may still claim a notification: one claimed now could
+   * finish before the run's timeout.
+   */
+  public canClaimAnotherNotification(now?: number): boolean {
+    return (
+      (now ?? Date.now()) - this.startedAt <=
+      SubscriberNotificationTiming.LATEST_CLAIM_IN_MS
+    );
+  }
+}
+
+export class SubscriberNotificationSendWindow {
+  public readonly closesAt: number;
+
+  public constructor(closesAt: number) {
+    this.closesAt = closesAt;
+  }
+
+  // Whether a new message may still be started.
+  public isOpen(now?: number): boolean {
+    return (now ?? Date.now()) < this.closesAt;
+  }
+}

package/Server/Utils/StatusPage/SubscriberTemplateIncidentRecordAccess.ts

@@ -0,0 +1,215 @@
+import Incident from "../../../Models/DatabaseModels/Incident";
+import IncidentCustomField from "../../../Models/DatabaseModels/IncidentCustomField";
+import BaseModel, {
+  DatabaseBaseModelType,
+} from "../../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
+import DatabaseCommonInteractionProps from "../../../Types/BaseDatabase/DatabaseCommonInteractionProps";
+import DatabaseCommonInteractionPropsUtil, {
+  PermissionType,
+} from "../../../Types/BaseDatabase/DatabaseCommonInteractionPropsUtil";
+import { CUSTOM_FIELD_TEMPLATE_VARIABLE_PREFIX } from "../../../Types/CustomField/CustomFieldVariableKey";
+import PermissionScope from "../../../Types/Database/AccessControl/PermissionScope";
+import NotAuthorizedException from "../../../Types/Exception/NotAuthorizedException";
+import Permission, {
+  PermissionHelper,
+  UserPermission,
+} from "../../../Types/Permission";
+
+/*
+ * WHO MAY PLACE INCIDENT RECORDS IN A SUBSCRIBER TEMPLATE.
+ *
+ * A custom subscriber notification template can place {{incidentLabels}} and
+ * any {{customFields.<key>}} - values from the team's incident records, which
+ * the status page does not show, marked for subscribers or not
+ * (SubscriberNotificationTemplateVariables.getIncidentRecordPlaceholders).
+ * Templates are written by the status page roles (StatusPageAdmin,
+ * StatusPageMember), which cannot read incidents; and a Slack, Teams or
+ * webhook subscriber can be added by the same roles, pointing at an address
+ * they own. Without this check such a role could send itself every
+ * incident's custom fields - guessing keys, which come predictably from
+ * field names, many at once, since a placeholder with no value is left as
+ * written.
+ *
+ * So a template may place them only if whoever writes it may read them
+ * directly: incidents, the column the value comes from (Incident.customFields
+ * or Incident.labels) and, for custom fields, the field definitions. And for
+ * every incident of the project, since a template reaches every incident
+ * that reaches its status pages: a grant limited to labels or to owned
+ * incidents does not count, and a block on any of those permissions refuses.
+ *
+ * The permissions are read from the models' own access control, so they
+ * follow any change to it. Root and master admin writes are not checked.
+ */
+
+interface AccessRequirement {
+  // What the caller must be able to read, for the refusal.
+  description: string;
+  permissions: Array<Permission>;
+}
+
+export default class SubscriberTemplateIncidentRecordAccess {
+  /**
+   * Refuses a write that places these placeholders (from
+   * getIncidentRecordPlaceholders) unless the caller may read what they
+   * read, for every incident of the project. Does nothing when there are
+   * none, or for root and master admin.
+   */
+  public static assertCanPlace(data: {
+    placeholders: Array<string>;
+    props: DatabaseCommonInteractionProps;
+  }): void {
+    if (
+      data.placeholders.length === 0 ||
+      data.props.isRoot ||
+      data.props.isMasterAdmin
+    ) {
+      return;
+    }
+
+    const unrestricted: Array<Permission> =
+      SubscriberTemplateIncidentRecordAccess.getUnrestrictedPermissions(
+        data.props,
+      );
+    const blocked: Array<Permission> =
+      DatabaseCommonInteractionPropsUtil.getUserPermissions(
+        data.props,
+        PermissionType.Block,
+      ).map((row: UserPermission): Permission => {
+        return row.permission;
+      });
+
+    for (const requirement of SubscriberTemplateIncidentRecordAccess.getRequirements(
+      data.placeholders,
+    )) {
+      const isGranted: boolean = PermissionHelper.doesPermissionsIntersect(
+        unrestricted,
+        requirement.permissions,
+      );
+      const isBlocked: boolean = PermissionHelper.doesPermissionsIntersect(
+        blocked,
+        requirement.permissions,
+      );
+
+      if (isGranted && !isBlocked) {
+        continue;
+      }
+
+      const placeholders: string = data.placeholders
+        .map((name: string): string => {
+          return `{{${name}}}`;
+        })
+        .join(", ");
+
+      throw new NotAuthorizedException(
+        `Placing ${placeholders} in a subscriber notification template needs permission to ${requirement.description} for every incident in the project: these values come from your team's incident records, which the status page does not show. You need one of these permissions, not limited to some labels or to incidents you own: ${PermissionHelper.getPermissionTitles(
+          requirement.permissions,
+        ).join(", ")}.`,
+      );
+    }
+  }
+
+  /*
+   * What these placeholders need: reading incidents, and the column each
+   * one reads - Incident.labels for {{incidentLabels}}; Incident.customFields
+   * and the field definitions for {{customFields.<key>}}.
+   */
+  public static getRequirements(
+    placeholders: Array<string>,
+  ): Array<AccessRequirement> {
+    const requirements: Array<AccessRequirement> = [
+      {
+        description: "read incidents",
+        permissions: this.getTableReadPermissions(Incident),
+      },
+    ];
+
+    if (
+      placeholders.some((name: string): boolean => {
+        return name.startsWith(CUSTOM_FIELD_TEMPLATE_VARIABLE_PREFIX);
+      })
+    ) {
+      requirements.push(
+        {
+          description: "read incident custom field values",
+          permissions: this.getColumnReadPermissions(Incident, "customFields"),
+        },
+        {
+          description: "read incident custom fields",
+          permissions: this.getTableReadPermissions(IncidentCustomField),
+        },
+      );
+    }
+
+    if (
+      placeholders.some((name: string): boolean => {
+        return !name.startsWith(CUSTOM_FIELD_TEMPLATE_VARIABLE_PREFIX);
+      })
+    ) {
+      requirements.push({
+        description: "read incident labels",
+        permissions: this.getColumnReadPermissions(Incident, "labels"),
+      });
+    }
+
+    return requirements;
+  }
+
+  /*
+   * The caller's granted permissions that reach every record, read as the
+   * read checks read them (PermissionHelper.getNonAccessControlPermissions,
+   * OwnedScopePermission): a row scoped to All, or a Labels (or legacy
+   * unscoped) row with no labels. A row limited to labels does not, nor one
+   * limited to owned records - except for a project-wide role that cannot
+   * be scoped (Project Owner, Project Admin), whose stray Owned scope the
+   * read checks ignore too.
+   */
+  private static getUnrestrictedPermissions(
+    props: DatabaseCommonInteractionProps,
+  ): Array<Permission> {
+    return DatabaseCommonInteractionPropsUtil.getUserPermissions(
+      props,
+      PermissionType.Allow,
+    )
+      .filter((row: UserPermission): boolean => {
+        if (row.scope === PermissionScope.All) {
+          return true;
+        }
+
+        if (row.scope === PermissionScope.Owned) {
+          return !PermissionHelper.isScopeApplicable(row.permission);
+        }
+
+        return !row.labelIds || row.labelIds.length === 0;
+      })
+      .map((row: UserPermission): Permission => {
+        return row.permission;
+      });
+  }
+
+  // As the table check reads them, the operational resource wildcard included.
+  private static getTableReadPermissions(
+    modelType: DatabaseBaseModelType,
+  ): Array<Permission> {
+    const model: BaseModel = new modelType();
+    const permissions: Array<Permission> = [...model.readRecordPermissions];
+
+    if (
+      model.isOperationalResource &&
+      !permissions.includes(Permission.ReadAllOperationalResources)
+    ) {
+      permissions.push(Permission.ReadAllOperationalResources);
+    }
+
+    return permissions;
+  }
+
+  // As the select check reads them: no wildcard reaches a column.
+  private static getColumnReadPermissions(
+    modelType: DatabaseBaseModelType,
+    columnName: string,
+  ): Array<Permission> {
+    return [
+      ...(new modelType().getColumnAccessControlFor(columnName)?.read || []),
+    ];
+  }
+}

package/Server/Utils/StatusPageResource.ts

@@ -1,5 +1,6 @@
 import StatusPageResource from "../../Models/DatabaseModels/StatusPageResource";
 import Dictionary from "../../Types/Dictionary";
+import SafeHtml from "../../Types/SafeHtml";
 
 export interface GetResourcesGroupedByGroupNameOptions {
   resources: Array<StatusPageResource>;
@@ -16,6 +17,12 @@ export const HTML_RESOURCE_GROUP_SEPARATOR: string = "<br/>";
  */
 export const PLAIN_TEXT_RESOURCE_GROUP_SEPARATOR: string = "; ";
 
+type FormatNameFunction = (name: string) => string;
+
+const asWritten: FormatNameFunction = (name: string): string => {
+  return name;
+};
+
 export default class StatusPageResourceUtil {
   /**
    * Formats an array of StatusPageResource items into an HTML string grouped by their resource group.
@@ -24,8 +31,18 @@ export default class StatusPageResourceUtil {
    * If resources are grouped, returns one group per line, like:
    * "EU: Infrastructure, Website<br/>UK: Infrastructure, API"
    *
+   * The result is HTML: every resource name, group name and the default
+   * value is HTML-escaped before it is joined, and the only markup is the
+   * "<br/>" between groups. A project member names resources and groups, and
+   * the default subscriber emails put this list into their raw-HTML slot, so
+   * a name such as `<a href="https://evil.example">Checkout</a>` would
+   * otherwise arrive as a live link. That is also what makes it safe to hand
+   * a custom email template as SafeHtml.
+   *
    * Use this only where the value is rendered as HTML (email bodies). Use
-   * getResourcesGroupedByGroupNameAsPlainText everywhere else.
+   * getResourcesGroupedByGroupNameAsPlainText everywhere else: it is not
+   * escaped, because SMS, Slack, Teams, subjects and webhooks show text as
+   * written.
    *
    * @param resources - Array of StatusPageResource items with displayName, statusPageGroupId, and optionally statusPageGroup.name
    * @param defaultValue - Value to return if no resources (defaults to "")
@@ -35,11 +52,12 @@ export default class StatusPageResourceUtil {
     resources: Array<StatusPageResource>,
     defaultValue: string = "",
   ): string {
-    return StatusPageResourceUtil.formatResourcesGroupedByGroupName(
-      resources,
-      defaultValue,
-      HTML_RESOURCE_GROUP_SEPARATOR,
-    );
+    return StatusPageResourceUtil.formatResourcesGroupedByGroupName({
+      resources: resources,
+      defaultValue: defaultValue,
+      groupSeparator: HTML_RESOURCE_GROUP_SEPARATOR,
+      formatName: SafeHtml.escape,
+    });
   }
 
   /**
@@ -55,18 +73,28 @@ export default class StatusPageResourceUtil {
     resources: Array<StatusPageResource>,
     defaultValue: string = "",
   ): string {
-    return StatusPageResourceUtil.formatResourcesGroupedByGroupName(
-      resources,
-      defaultValue,
-      PLAIN_TEXT_RESOURCE_GROUP_SEPARATOR,
-    );
+    return StatusPageResourceUtil.formatResourcesGroupedByGroupName({
+      resources: resources,
+      defaultValue: defaultValue,
+      groupSeparator: PLAIN_TEXT_RESOURCE_GROUP_SEPARATOR,
+      formatName: asWritten,
+    });
   }
 
-  private static formatResourcesGroupedByGroupName(
-    resources: Array<StatusPageResource>,
-    defaultValue: string,
-    groupSeparator: string,
-  ): string {
+  /*
+   * Groups are keyed by the name as written, so two groups are merged only
+   * when their names really are the same; formatName (the HTML escaping, for
+   * the HTML form) is applied to each piece of text as it is written out.
+   */
+  private static formatResourcesGroupedByGroupName(data: {
+    resources: Array<StatusPageResource>;
+    defaultValue: string;
+    groupSeparator: string;
+    formatName: FormatNameFunction;
+  }): string {
+    const { resources, groupSeparator, formatName } = data;
+    const defaultValue: string = formatName(data.defaultValue);
+
     if (!resources || resources.length === 0) {
       return defaultValue;
     }
@@ -83,9 +111,10 @@ export default class StatusPageResourceUtil {
           .map((r: StatusPageResource) => {
             return r.displayName;
           })
-          .filter((name: string | undefined) => {
-            return name;
+          .filter((name: string | undefined): name is string => {
+            return Boolean(name);
           })
+          .map(formatName)
           .join(", ") || ""
       );
     }
@@ -119,13 +148,15 @@ export default class StatusPageResourceUtil {
     // Add grouped resources
     for (const groupName in resourcesByGroup) {
       const groupResources: Array<string> = resourcesByGroup[groupName]!;
-      formattedGroups.push(`${groupName}: ${groupResources.join(", ")}`);
+      formattedGroups.push(
+        `${formatName(groupName)}: ${groupResources.map(formatName).join(", ")}`,
+      );
     }
 
     // Add ungrouped resources on separate lines (without "Other" label)
     if (ungroupedResources.length > 0) {
       for (const resourceName of ungroupedResources) {
-        formattedGroups.push(resourceName);
+        formattedGroups.push(formatName(resourceName));
       }
     }