@oneuptime/common 14.0.6 → 14.0.8

package/Server/Utils/StatusPage/SubscriberNotificationResendAccess.ts

@@ -0,0 +1,393 @@
+import BaseModel from "../../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
+import DatabaseCommonInteractionProps from "../../../Types/BaseDatabase/DatabaseCommonInteractionProps";
+import LIMIT_MAX from "../../../Types/Database/LimitMax";
+import BadDataException from "../../../Types/Exception/BadDataException";
+import NotAuthorizedException from "../../../Types/Exception/NotAuthorizedException";
+import { JSONObject } from "../../../Types/JSON";
+import StatusPageSubscriberNotificationStatus from "../../../Types/StatusPage/StatusPageSubscriberNotificationStatus";
+import SubscriberNotificationResend from "../../../Types/StatusPage/SubscriberNotificationResend";
+import DatabaseService from "../../Services/DatabaseService";
+import DatabaseRequestType from "../../Types/BaseDatabase/DatabaseRequestType";
+import Query from "../../Types/Database/Query";
+import Select from "../../Types/Database/Select";
+import UpdateBy from "../../Types/Database/UpdateBy";
+import ColumnPermissions from "../../Types/Database/Permissions/ColumnPermission";
+import ModelPermission from "../../Types/Database/Permissions/Index";
+import TablePermission from "../../Types/Database/Permissions/TablePermission";
+
+/*
+ * Who may send a status page subscriber notification again, and when.
+ *
+ * Sending one again is a write: its status goes back to Pending, and the
+ * job that sends it picks it up. So the update's own permission check - the
+ * table's update access control and each written column's - already decides
+ * who may. For a public note that is not enough: the note's edit permission
+ * is granted on its own, and editing a note does not make someone its
+ * poster. Sending a note's notification again tells every subscriber of the
+ * incident's status pages what the note says, which is what posting it did,
+ * so it also needs the permission to post a note that notifies subscribers.
+ * The same goes for telling subscribers about an edit (the note's 'updated'
+ * notification): an editor can change the text first, so with the edit
+ * permission alone it would be a way to message every subscriber.
+ *
+ * When: never while the notification is being sent. The send settles its
+ * own status when it finishes, so a Pending written over a running send
+ * either lets a second run send it at the same time - the same subscribers
+ * messaged twice, each run then overwriting the other's status - or is
+ * overwritten and lost without a word. That holds for every notification a
+ * user can send again (assertNotQueuedWhileBeingSent).
+ *
+ * These checks run in the services' onBeforeUpdate, before the update's own
+ * check (DatabaseService runs that after the hooks), so each one checks
+ * permissions before it reads anything: a caller who may not send a
+ * notification again is told only that, never what state the notification
+ * is in. Rows are then read with the caller's own permissions, so nothing
+ * they cannot read decides the answer either.
+ *
+ * Root callers - the workers and other internal writes - are trusted and
+ * never checked. Master admins skip the permission checks, as they do
+ * everywhere, but not the checks on the notification's state.
+ */
+export default class SubscriberNotificationResendAccess {
+  /*
+   * The columns of a public note that sending its 'posted' notification
+   * again writes.
+   */
+  private static readonly PUBLIC_NOTE_RESEND_COLUMNS: JSONObject = {
+    subscriberNotificationStatusOnNoteCreated:
+      StatusPageSubscriberNotificationStatus.Pending,
+    subscriberNotificationStatusMessage: "",
+  };
+
+  /*
+   * The columns of a public note that telling subscribers about an edit
+   * writes.
+   */
+  private static readonly PUBLIC_NOTE_UPDATE_NOTIFICATION_COLUMNS: JSONObject =
+    {
+      subscriberNotificationStatusOnNoteUpdated:
+        StatusPageSubscriberNotificationStatus.Pending,
+      subscriberNotificationStatusMessageOnNoteUpdated: "",
+    };
+
+  public static isPermissionChecked(
+    props: DatabaseCommonInteractionProps,
+  ): boolean {
+    return !props.isRoot && !props.isMasterAdmin;
+  }
+
+  /*
+   * Throws `refusal` unless the caller may write these columns of the model:
+   * the table's update access control, then each column's. A request that
+   * fails for any other reason - an expired session, an unpaid plan - keeps
+   * its own error.
+   */
+  public static assertCallerMayUpdateColumns<
+    TBaseModel extends BaseModel,
+  >(data: {
+    modelType: { new (): TBaseModel };
+    columns: JSONObject;
+    props: DatabaseCommonInteractionProps;
+    refusal: string;
+  }): void {
+    if (!this.isPermissionChecked(data.props)) {
+      return;
+    }
+
+    this.withRefusal(data.refusal, () => {
+      TablePermission.checkTableLevelPermissions(
+        data.modelType,
+        data.props,
+        DatabaseRequestType.Update,
+      );
+
+      ColumnPermissions.checkDataColumnPermissions(
+        data.modelType,
+        data.columns as unknown as TBaseModel,
+        data.props,
+        DatabaseRequestType.Update,
+      );
+    });
+  }
+
+  /*
+   * Throws `refusal` unless the caller may post a new note of this type
+   * that notifies subscribers: the create check a post with Notify Status
+   * Page Subscribers ticked goes through, on a note carrying only that
+   * choice.
+   */
+  public static assertCallerMayPostNotifyingNote<
+    TNote extends BaseModel,
+  >(data: {
+    modelType: { new (): TNote };
+    props: DatabaseCommonInteractionProps;
+    refusal: string;
+  }): void {
+    if (!this.isPermissionChecked(data.props)) {
+      return;
+    }
+
+    const probe: TNote = new data.modelType();
+    (probe as unknown as JSONObject)[
+      "shouldStatusPageSubscribersBeNotifiedOnNoteCreated"
+    ] = true;
+
+    this.withRefusal(data.refusal, () => {
+      ModelPermission.checkCreatePermissions(data.modelType, probe, data.props);
+    });
+  }
+
+  /*
+   * A user's or an API key's request to send a public note's 'posted'
+   * notification again - an update writing Pending into
+   * subscriberNotificationStatusOnNoteCreated, which is what the dashboard's
+   * Retry and Resend do. Shared by the incident, incident episode and
+   * scheduled maintenance public notes.
+   *
+   * - The caller must be able to edit the note's notification status and to
+   *   post a note that notifies subscribers (see the header).
+   * - Every note the update matches must be one whose notification can be
+   *   sent again (SubscriberNotificationResend.getPublicNoteResendRefusal):
+   *   not a note posted without notifying subscribers, which the job never
+   *   sends, and not one being sent right now, whose send would overwrite
+   *   the request when it finishes.
+   *
+   * An update that does not ask for it is not looked at.
+   */
+  public static async assertPublicNoteResendAllowed<
+    TNote extends BaseModel,
+  >(data: {
+    modelType: { new (): TNote };
+    service: DatabaseService<TNote>;
+    updateBy: UpdateBy<TNote>;
+  }): Promise<void> {
+    const updateBy: UpdateBy<TNote> = data.updateBy;
+
+    if (
+      updateBy.props.isRoot ||
+      !SubscriberNotificationResend.isPublicNoteResendRequested(
+        updateBy.data as {
+          subscriberNotificationStatusOnNoteCreated?: unknown;
+        },
+      )
+    ) {
+      return;
+    }
+
+    this.assertCallerMayUpdateColumns({
+      modelType: data.modelType,
+      columns: this.PUBLIC_NOTE_RESEND_COLUMNS,
+      props: updateBy.props,
+      refusal: SubscriberNotificationResend.noPermissionToResendNoteMessage,
+    });
+
+    this.assertCallerMayPostNotifyingNote({
+      modelType: data.modelType,
+      props: updateBy.props,
+      refusal: SubscriberNotificationResend.noPermissionToResendNoteMessage,
+    });
+
+    const notes: Array<TNote> = await data.service.findBy({
+      query: updateBy.query as Query<TNote>,
+      select: {
+        _id: true,
+        subscriberNotificationStatusOnNoteCreated: true,
+        shouldStatusPageSubscribersBeNotifiedOnNoteCreated: true,
+      } as unknown as Select<TNote>,
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: updateBy.props,
+    });
+
+    for (const note of notes) {
+      const record: JSONObject = note as unknown as JSONObject;
+
+      const refusal: string | null =
+        SubscriberNotificationResend.getPublicNoteResendRefusal({
+          status: record["subscriberNotificationStatusOnNoteCreated"] as
+            | StatusPageSubscriberNotificationStatus
+            | undefined,
+          shouldStatusPageSubscribersBeNotifiedOnNoteCreated: record[
+            "shouldStatusPageSubscribersBeNotifiedOnNoteCreated"
+          ] as boolean | undefined,
+        });
+
+      if (refusal) {
+        throw new BadDataException(refusal);
+      }
+    }
+  }
+
+  /*
+   * A user's or an API key's request to tell subscribers about an edit to a
+   * public note: the edit carries the notify-on-edit misc data prop
+   * (SubscriberUpdateNotification, which the service has turned into a
+   * Pending by now), or writes Pending into
+   * subscriberNotificationStatusOnNoteUpdated itself, as the dashboard's
+   * Retry of a failed update does. Shared by the incident, incident episode
+   * and scheduled maintenance public notes.
+   *
+   * - The caller must be able to write the update notification's status and
+   *   to post a note that notifies subscribers (see the header).
+   * - It is refused while the update notification is being sent
+   *   (assertNotQueuedWhileBeingSent): the edit can be saved without it.
+   *
+   * An update that does not ask for it is not looked at.
+   */
+  public static async assertPublicNoteUpdateNotificationAllowed<
+    TNote extends BaseModel,
+  >(data: {
+    modelType: { new (): TNote };
+    service: DatabaseService<TNote>;
+    updateBy: UpdateBy<TNote>;
+  }): Promise<void> {
+    const updateBy: UpdateBy<TNote> = data.updateBy;
+
+    if (
+      updateBy.props.isRoot ||
+      (updateBy.data as JSONObject)[
+        "subscriberNotificationStatusOnNoteUpdated"
+      ] !== StatusPageSubscriberNotificationStatus.Pending
+    ) {
+      return;
+    }
+
+    this.assertCallerMayUpdateColumns({
+      modelType: data.modelType,
+      columns: this.PUBLIC_NOTE_UPDATE_NOTIFICATION_COLUMNS,
+      props: updateBy.props,
+      refusal:
+        SubscriberNotificationResend.noPermissionToNotifyAboutEditMessage,
+    });
+
+    this.assertCallerMayPostNotifyingNote({
+      modelType: data.modelType,
+      props: updateBy.props,
+      refusal:
+        SubscriberNotificationResend.noPermissionToNotifyAboutEditMessage,
+    });
+
+    await this.assertNotQueuedWhileBeingSent({
+      modelType: data.modelType,
+      service: data.service,
+      updateBy: updateBy,
+      statusColumns: ["subscriberNotificationStatusOnNoteUpdated"],
+      refusal: SubscriberNotificationResend.updateBeingSentMessage,
+    });
+  }
+
+  /*
+   * A user's or an API key's update writing Pending into a notification's
+   * status column - Retry, Resend, or the API route of sending it again -
+   * while that notification is being sent (InProgress). Refused with
+   * `refusal` (SubscriberNotificationResend.beingSentMessage by default):
+   * see the header for why.
+   *
+   * Only the columns the update writes Pending into are looked at, so an
+   * edit that leaves the notification alone is never refused, and Pending
+   * written over Pending (queued already) changes nothing and is let
+   * through.
+   *
+   * Permission first: a caller who may not write those columns is left to
+   * the update's own check, which refuses them anyway, and never learns the
+   * notification's state from this one. The rows are then read with the
+   * caller's own permissions, so nothing they cannot read decides the
+   * answer. Root callers - the workers - are never checked.
+   */
+  public static async assertNotQueuedWhileBeingSent<
+    TBaseModel extends BaseModel,
+  >(data: {
+    modelType: { new (): TBaseModel };
+    service: DatabaseService<TBaseModel>;
+    updateBy: UpdateBy<TBaseModel>;
+    statusColumns: Array<string>;
+    refusal?: string | undefined;
+  }): Promise<void> {
+    const updateBy: UpdateBy<TBaseModel> = data.updateBy;
+
+    if (updateBy.props.isRoot) {
+      return;
+    }
+
+    const written: JSONObject = (updateBy.data || {}) as JSONObject;
+
+    const queuedColumns: Array<string> = data.statusColumns.filter(
+      (column: string): boolean => {
+        return (
+          written[column] === StatusPageSubscriberNotificationStatus.Pending
+        );
+      },
+    );
+
+    if (queuedColumns.length === 0) {
+      return;
+    }
+
+    const pendingColumns: JSONObject = {};
+    const select: JSONObject = { _id: true };
+
+    for (const column of queuedColumns) {
+      pendingColumns[column] = StatusPageSubscriberNotificationStatus.Pending;
+      select[column] = true;
+    }
+
+    try {
+      this.assertCallerMayUpdateColumns({
+        modelType: data.modelType,
+        columns: pendingColumns,
+        props: updateBy.props,
+        refusal: "",
+      });
+    } catch (err) {
+      if (err instanceof NotAuthorizedException) {
+        return;
+      }
+
+      throw err;
+    }
+
+    const rows: Array<TBaseModel> = await data.service.findBy({
+      query: updateBy.query as Query<TBaseModel>,
+      select: select as unknown as Select<TBaseModel>,
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: updateBy.props,
+    });
+
+    for (const row of rows) {
+      const record: JSONObject = row as unknown as JSONObject;
+
+      for (const column of queuedColumns) {
+        if (
+          record[column] === StatusPageSubscriberNotificationStatus.InProgress
+        ) {
+          throw new BadDataException(
+            data.refusal || SubscriberNotificationResend.beingSentMessage,
+          );
+        }
+      }
+    }
+  }
+
+  /*
+   * Runs a permission check and turns its refusal into `refusal`, which says
+   * what the caller was trying to do. A column refusal is a BadDataException
+   * ("User is not allowed to update on ... column"), a table refusal a
+   * NotAuthorizedException; both become a NotAuthorizedException. Anything
+   * else - an anonymous caller, an unpaid plan - passes through as it is.
+   */
+  private static withRefusal(refusal: string, check: () => void): void {
+    try {
+      check();
+    } catch (err) {
+      if (
+        err instanceof NotAuthorizedException ||
+        err instanceof BadDataException
+      ) {
+        throw new NotAuthorizedException(refusal);
+      }
+
+      throw err;
+    }
+  }
+}

package/Server/Utils/StatusPage/SubscriberNotificationRunLimit.ts

@@ -0,0 +1,246 @@
+import OneUptimeDate from "../../../Types/Date";
+import ObjectID from "../../../Types/ObjectID";
+import Semaphore, {
+  SemaphoreLockTimeoutError,
+  SemaphorePermit,
+} from "../../Infrastructure/Semaphore";
+import logger from "../Logger";
+
+/*
+ * How many runs of one subscriber job may be sending at once, across every
+ * worker.
+ *
+ * The incident and episode subscriber jobs run every minute on the shared
+ * Worker queue, and each run awaits the notifications it claims - minutes,
+ * for a big or slow send (SubscriberNotificationTiming). The runs overlap by
+ * design, and nothing bounded how many: with a backlog of slow sends - a
+ * tenant whose mail server answers slowly, a large outage with many notes and
+ * state changes on big pages - every job could keep a run going every minute
+ * for the length of its timeout, some 25-30 runs per job, nine jobs, all
+ * holding slots of the one Worker consumer (WORKER_CONCURRENCY, 100 by
+ * default) that on-call escalation, monitors and every other cron share.
+ * Exactly during an outage, subscriber email could starve paging.
+ *
+ * So a run takes a permit first: at most MAX_CONCURRENT_RUNS_PER_JOB runs of
+ * a job hold one at a time, nine jobs times that in all. A run that finds
+ * them all taken returns at once, leaving its notifications Pending for the
+ * runs that follow; a run's claim is atomic (SubscriberNotificationClaim), so
+ * which run sends a notification never matters. A permit is refreshed while
+ * its run lives and lapses PERMIT_TTL_IN_MS after a worker dies holding it,
+ * so a crash frees its slot within minutes, not at the job timeout.
+ *
+ * If Redis cannot be reached for the permit, the run goes ahead without one:
+ * the queue that started it runs on Redis too, so that is a blip, and sending
+ * the notifications matters more than bounding them for its length.
+ *
+ * One project cannot take them all. The permits are shared by every project
+ * on the server, and a run sends one notification at a time, so a tenant
+ * whose own mail server hangs - each message waiting out its timeout - or who
+ * posts a burst of notes on a large page would otherwise hold every run of
+ * the job, and every other tenant's notifications of that kind would wait
+ * behind it. So a run also takes one of the project's slots for the job
+ * before it claims each notification (takeProjectSlot): at most
+ * MAX_CONCURRENT_SENDS_PER_PROJECT of one project's notifications of a job
+ * are being sent at once, and a run that finds the project's slots taken
+ * leaves that notification Pending and goes on to the next project's.
+ *
+ * Both numbers can be raised for a larger fleet:
+ * SUBSCRIBER_NOTIFICATION_MAX_CONCURRENT_RUNS_PER_JOB and
+ * SUBSCRIBER_NOTIFICATION_MAX_CONCURRENT_SENDS_PER_PROJECT.
+ */
+
+// A slot one project holds while one of its notifications is being sent.
+export interface SubscriberNotificationProjectSlot {
+  release: () => Promise<void>;
+}
+
+const parsePositiveIntFromEnv: (envKey: string, fallback: number) => number = (
+  envKey: string,
+  fallback: number,
+): number => {
+  const rawValue: string | undefined = process.env[envKey];
+
+  if (!rawValue) {
+    return fallback;
+  }
+
+  const parsedValue: number = parseInt(rawValue, 10);
+
+  if (!Number.isFinite(parsedValue) || parsedValue <= 0) {
+    return fallback;
+  }
+
+  return parsedValue;
+};
+
+// Held when there is nothing to bound: no project, or no Redis.
+const NO_SLOT: SubscriberNotificationProjectSlot = {
+  release: async (): Promise<void> => {},
+};
+
+export default class SubscriberNotificationRunLimit {
+  public static readonly MAX_CONCURRENT_RUNS_PER_JOB: number =
+    parsePositiveIntFromEnv(
+      "SUBSCRIBER_NOTIFICATION_MAX_CONCURRENT_RUNS_PER_JOB",
+      4,
+    );
+
+  /*
+   * Fewer than the runs of a job, so a project whose sends are slow always
+   * leaves some of them to everyone else.
+   */
+  public static readonly MAX_CONCURRENT_SENDS_PER_PROJECT: number = Math.max(
+    1,
+    Math.min(
+      parsePositiveIntFromEnv(
+        "SUBSCRIBER_NOTIFICATION_MAX_CONCURRENT_SENDS_PER_PROJECT",
+        2,
+      ),
+      SubscriberNotificationRunLimit.MAX_CONCURRENT_RUNS_PER_JOB - 1,
+    ),
+  );
+
+  // The per-project slots' Redis namespace; the key is the job and the project.
+  public static readonly PROJECT_NAMESPACE: string =
+    "subscriber-notification-project";
+
+  public static readonly PERMIT_TTL_IN_MS: number =
+    OneUptimeDate.convertMinutesToMilliseconds(2);
+
+  // The permits' Redis namespace; the key is the job's name.
+  public static readonly NAMESPACE: string = "subscriber-notification-run";
+
+  /*
+   * The job's run, run only while it holds one of the job's permits. For
+   * RunCron: RunCron("Job:Name", options, limit("Job:Name", async () => {})).
+   */
+  public static limit(
+    jobName: string,
+    run: () => Promise<void>,
+  ): () => Promise<void> {
+    return async (): Promise<void> => {
+      await SubscriberNotificationRunLimit.runWithinLimit(jobName, run);
+    };
+  }
+
+  /*
+   * One of the project's slots for this job, taken before a run claims one
+   * of the project's notifications; release it once that notification has
+   * settled. Null when the project already has
+   * MAX_CONCURRENT_SENDS_PER_PROJECT of this job's notifications being sent:
+   * leave this one Pending for a later run, and go on to the next. Never
+   * throws: without a project, or without Redis, the send goes ahead
+   * unbounded, as a run does without its permit.
+   */
+  public static async takeProjectSlot(data: {
+    jobName: string;
+    projectId: ObjectID | string | undefined | null;
+  }): Promise<SubscriberNotificationProjectSlot | null> {
+    const projectId: string = data.projectId?.toString().trim() || "";
+
+    if (!projectId) {
+      return NO_SLOT;
+    }
+
+    let permit: SemaphorePermit;
+
+    try {
+      permit = await Semaphore.acquirePermit({
+        key: `${data.jobName}:${projectId.toLowerCase()}`,
+        namespace: SubscriberNotificationRunLimit.PROJECT_NAMESPACE,
+        limit: SubscriberNotificationRunLimit.MAX_CONCURRENT_SENDS_PER_PROJECT,
+        lockTimeout: SubscriberNotificationRunLimit.PERMIT_TTL_IN_MS,
+        acquireAttemptsLimit: 1,
+        acquireTimeout: 5000,
+        retryInterval: 50,
+        onLockLost: (err: Error): void => {
+          logger.warn(
+            `${data.jobName}: lost the slot of project ${projectId} while sending: ${err.message}`,
+          );
+        },
+      });
+    } catch (err) {
+      if (err instanceof SemaphoreLockTimeoutError) {
+        logger.debug(
+          `${data.jobName}: project ${projectId} already has ${SubscriberNotificationRunLimit.MAX_CONCURRENT_SENDS_PER_PROJECT} notification(s) being sent; leaving this one Pending.`,
+        );
+        return null;
+      }
+
+      logger.warn(
+        `${data.jobName}: could not take a slot for project ${projectId} (${err instanceof Error ? err.message : String(err)}); sending without one.`,
+      );
+      return NO_SLOT;
+    }
+
+    return {
+      release: async (): Promise<void> => {
+        try {
+          await Semaphore.releasePermit(permit);
+        } catch (err) {
+          // It lapses on its own after PERMIT_TTL_IN_MS.
+          logger.warn(
+            `${data.jobName}: could not release the slot of project ${projectId}: ${err instanceof Error ? err.message : String(err)}`,
+          );
+        }
+      },
+    };
+  }
+
+  public static async runWithinLimit(
+    jobName: string,
+    run: () => Promise<void>,
+  ): Promise<void> {
+    let permit: SemaphorePermit | null = null;
+
+    try {
+      permit = await Semaphore.acquirePermit({
+        key: jobName,
+        namespace: SubscriberNotificationRunLimit.NAMESPACE,
+        limit: SubscriberNotificationRunLimit.MAX_CONCURRENT_RUNS_PER_JOB,
+        lockTimeout: SubscriberNotificationRunLimit.PERMIT_TTL_IN_MS,
+        // One look: a full set of permits means other runs are sending.
+        acquireAttemptsLimit: 1,
+        acquireTimeout: 5000,
+        retryInterval: 50,
+        onLockLost: (err: Error): void => {
+          /*
+           * Redis lost the permit (it could not be refreshed). The run goes
+           * on - its claims keep its notifications its own - and another
+           * run may take the freed slot meanwhile.
+           */
+          logger.warn(
+            `${jobName}: lost its run permit while sending: ${err.message}`,
+          );
+        },
+      });
+    } catch (err) {
+      if (err instanceof SemaphoreLockTimeoutError) {
+        logger.debug(
+          `${jobName}: ${SubscriberNotificationRunLimit.MAX_CONCURRENT_RUNS_PER_JOB} runs are already sending; leaving the Pending notifications to the runs that follow.`,
+        );
+        return;
+      }
+
+      logger.warn(
+        `${jobName}: could not take a run permit (${err instanceof Error ? err.message : String(err)}); running without one.`,
+      );
+      permit = null;
+    }
+
+    try {
+      await run();
+    } finally {
+      if (permit) {
+        try {
+          await Semaphore.releasePermit(permit);
+        } catch (err) {
+          // It lapses on its own after PERMIT_TTL_IN_MS.
+          logger.warn(
+            `${jobName}: could not release its run permit: ${err instanceof Error ? err.message : String(err)}`,
+          );
+        }
+      }
+    }
+  }
+}