@oneuptime/common 14.0.7 → 14.0.9

package/Server/Middleware/SubscriberNotificationPreviewRateLimit.ts

@@ -0,0 +1,237 @@
+import Redis, { ClientType } from "../Infrastructure/Redis";
+import {
+  ExpressRequest,
+  ExpressResponse,
+  NextFunction,
+} from "../Utils/Express";
+import logger from "../Utils/Logger";
+import Response from "../Utils/Response";
+import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
+import SubscriberNotificationTestSendRateLimit from "./SubscriberNotificationTestSendRateLimit";
+
+/*
+ * Request limiting for "Preview notification" (POST /preview on the
+ * subscriber notification preview router).
+ *
+ * A preview sends nothing, but it is not free: it renders the incident's
+ * description and custom fields once, then an email for every status page
+ * the caller can read, and returns all of that HTML in one answer. Any
+ * project member holding one of the audience roles may call it, as often as
+ * they like, so this bounds how much rendering one person can ask the App
+ * server for. The request itself is bounded too
+ * (SubscriberNotificationPreviewBuilder.parseRequest caps the text and the
+ * custom field values).
+ *
+ * The limit is loose: someone writing an incident or a note and previewing
+ * it after each change is nowhere near it. The counter is per user, across
+ * every project and incident, like the test email's
+ * (SubscriberNotificationTestSendRateLimit, whose user key this shares).
+ *
+ * THIS FAILS OPEN
+ *
+ * If Redis is unreachable the preview is served. That is the opposite of the
+ * test email's choice, and the difference is what is at stake: a test email
+ * is real mail, and this counter would be the only bound on it; a preview
+ * sends nothing, its request is size-capped, and refusing it would take the
+ * preview away from everyone for as long as Redis is down.
+ */
+
+export enum SubscriberNotificationPreviewRateLimitOutcome {
+  Allowed = "allowed",
+  RateLimited = "rate-limited",
+
+  // Redis is unreachable: served, since this fails open.
+  CounterUnavailable = "counter-unavailable",
+}
+
+export interface SubscriberNotificationPreviewRateLimitDecision {
+  outcome: SubscriberNotificationPreviewRateLimitOutcome;
+  retryAfterSeconds?: number | undefined;
+  // True only on the request that first crossed the line in this window.
+  isFirstRejectionInWindow?: boolean | undefined;
+}
+
+export interface SubscriberNotificationPreviewRateLimitConfig {
+  windowSeconds: number;
+  perUserLimit: number;
+}
+
+const KEY_PREFIX: string = "subnotifpreview:rl:";
+
+// Counter keys outlive their window by one full window.
+const TTL_MULTIPLIER: number = 2;
+
+// How often the "counter unavailable" condition may be logged.
+const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS: number = 60 * 1000;
+
+export default class SubscriberNotificationPreviewRateLimit {
+  // Sixty previews every ten minutes: one every ten seconds, sustained.
+  private static readonly config: SubscriberNotificationPreviewRateLimitConfig =
+    {
+      windowSeconds: 10 * 60,
+      perUserLimit: 60,
+    };
+
+  private static counterUnavailableLastLoggedAt: number = 0;
+
+  // The limit in force, for tests and for operators' diagnostics.
+  public static getConfig(): SubscriberNotificationPreviewRateLimitConfig {
+    return { ...SubscriberNotificationPreviewRateLimit.config };
+  }
+
+  /*
+   * Count this request against the user and decide. A rejected request is
+   * counted too, so a caller who keeps trying keeps their window pinned.
+   */
+  public static async consume(data: {
+    userKey: string;
+  }): Promise<SubscriberNotificationPreviewRateLimitDecision> {
+    const client: ClientType | null = Redis.getClient();
+
+    if (!client || !Redis.isConnected()) {
+      return {
+        outcome:
+          SubscriberNotificationPreviewRateLimitOutcome.CounterUnavailable,
+      };
+    }
+
+    const config: SubscriberNotificationPreviewRateLimitConfig =
+      SubscriberNotificationPreviewRateLimit.config;
+    const windowMs: number = config.windowSeconds * 1000;
+    const windowIndex: number = Math.floor(Date.now() / windowMs);
+    const counterKey: string = `${KEY_PREFIX}u:${data.userKey}:${windowIndex}`;
+
+    try {
+      const count: number = await client.incr(counterKey);
+
+      if (typeof count !== "number") {
+        throw new Error("Rate limit counter returned a non-numeric value");
+      }
+
+      // Set only by the request that created the key, so the window is fixed.
+      if (count === 1) {
+        await client.expire(counterKey, config.windowSeconds * TTL_MULTIPLIER);
+      }
+
+      if (count > config.perUserLimit) {
+        return {
+          outcome: SubscriberNotificationPreviewRateLimitOutcome.RateLimited,
+          retryAfterSeconds:
+            SubscriberNotificationTestSendRateLimit.getSecondsUntilWindowEnd(
+              config.windowSeconds,
+            ),
+          isFirstRejectionInWindow: count === config.perUserLimit + 1,
+        };
+      }
+
+      return {
+        outcome: SubscriberNotificationPreviewRateLimitOutcome.Allowed,
+      };
+    } catch (err) {
+      if (
+        SubscriberNotificationPreviewRateLimit.shouldLogCounterUnavailable()
+      ) {
+        logger.warn(
+          `SubscriberNotificationPreviewRateLimit: counter failed for user ${data.userKey}`,
+        );
+        logger.warn(err);
+      }
+
+      return {
+        outcome:
+          SubscriberNotificationPreviewRateLimitOutcome.CounterUnavailable,
+      };
+    }
+  }
+
+  /*
+   * Express middleware. Register after UserMiddleware (the counter is per
+   * user) and before the handler, so a flood is refused before it renders
+   * anything.
+   */
+  public static getMiddleware(): (
+    req: ExpressRequest,
+    res: ExpressResponse,
+    next: NextFunction,
+  ) => Promise<void> {
+    return async (
+      req: ExpressRequest,
+      res: ExpressResponse,
+      next: NextFunction,
+    ): Promise<void> => {
+      const userKey: string =
+        SubscriberNotificationTestSendRateLimit.resolveUserKey(req);
+
+      const decision: SubscriberNotificationPreviewRateLimitDecision =
+        await SubscriberNotificationPreviewRateLimit.consume({
+          userKey: userKey,
+        });
+
+      if (
+        decision.outcome ===
+        SubscriberNotificationPreviewRateLimitOutcome.RateLimited
+      ) {
+        if (decision.retryAfterSeconds) {
+          const setHeader: unknown = (
+            res as unknown as Record<string, unknown>
+          )["setHeader"];
+
+          if (typeof setHeader === "function") {
+            (setHeader as (name: string, value: string) => void).call(
+              res,
+              "Retry-After",
+              String(decision.retryAfterSeconds),
+            );
+          }
+        }
+
+        if (decision.isFirstRejectionInWindow) {
+          logger.warn(
+            `SubscriberNotificationPreviewRateLimit: rejected previews from user ${userKey} for the rest of this window`,
+          );
+        }
+
+        return Response.sendErrorResponse(
+          req,
+          res,
+          new TooManyRequestsException(
+            "You have previewed notifications too many times in a short while. Please try again in a few minutes.",
+          ),
+        );
+      }
+
+      if (
+        decision.outcome ===
+          SubscriberNotificationPreviewRateLimitOutcome.CounterUnavailable &&
+        SubscriberNotificationPreviewRateLimit.shouldLogCounterUnavailable()
+      ) {
+        // Fails open: see the note at the top of this file.
+        logger.warn(
+          "SubscriberNotificationPreviewRateLimit: rate limit counter unavailable, serving previews without a limit",
+        );
+      }
+
+      return next();
+    };
+  }
+
+  /*
+   * A Redis outage means every request takes the unavailable path; one log
+   * line per interval makes the condition visible without burying the rest.
+   */
+  private static shouldLogCounterUnavailable(): boolean {
+    const now: number = Date.now();
+
+    if (
+      now -
+        SubscriberNotificationPreviewRateLimit.counterUnavailableLastLoggedAt <
+      COUNTER_UNAVAILABLE_LOG_INTERVAL_MS
+    ) {
+      return false;
+    }
+
+    SubscriberNotificationPreviewRateLimit.counterUnavailableLastLoggedAt = now;
+
+    return true;
+  }
+}

package/Server/Middleware/SubscriberNotificationTestSendRateLimit.ts

@@ -0,0 +1,304 @@
+import Redis, { ClientType } from "../Infrastructure/Redis";
+import {
+  ExpressRequest,
+  ExpressResponse,
+  NextFunction,
+  OneUptimeRequest,
+} from "../Utils/Express";
+import logger from "../Utils/Logger";
+import Response from "../Utils/Response";
+import ObjectID from "../../Types/ObjectID";
+import ServiceUnavailableException from "../../Types/Exception/ServiceUnavailableException";
+import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
+
+/*
+ * Request limiting for "Send test to me" (POST /send-test on the subscriber
+ * notification preview router), which emails the signed-in user the status
+ * page subscriber email they are previewing.
+ *
+ * The route only ever sends to the caller's own verified account email, so
+ * what this bounds is volume: every allowed request is a real email, through
+ * the status page's own SMTP server or the instance's mail provider, and
+ * nothing else stops a script (or a held-down button) from sending hundreds.
+ * The counter is per user, across every project, status page and incident:
+ * a user is one inbox.
+ *
+ * THIS FAILS CLOSED
+ *
+ * If Redis is unreachable, the test is refused with "try again later". That
+ * is the opposite of the choice VerificationCodeRateLimit makes, and the
+ * difference is what else bounds the requests: there, the code's expiry, its
+ * attempt counter and the resend cooldown all live in Postgres and stay in
+ * force; here this counter is the only bound on how much mail is sent. A
+ * test email is a convenience, and a Redis blip costs nothing but a retry.
+ */
+
+export enum SubscriberNotificationTestSendRateLimitOutcome {
+  Allowed = "allowed",
+  RateLimited = "rate-limited",
+
+  // Redis is unreachable, so the limit cannot be honoured.
+  CounterUnavailable = "counter-unavailable",
+}
+
+export interface SubscriberNotificationTestSendRateLimitDecision {
+  outcome: SubscriberNotificationTestSendRateLimitOutcome;
+  retryAfterSeconds?: number | undefined;
+  /*
+   * True only on the request that first crossed the line in this window, so
+   * a flood produces one log line per user per window.
+   */
+  isFirstRejectionInWindow?: boolean | undefined;
+}
+
+export interface SubscriberNotificationTestSendRateLimitConfig {
+  windowSeconds: number;
+  perUserLimit: number;
+}
+
+const KEY_PREFIX: string = "subnotiftest:rl:";
+
+/*
+ * Counter keys outlive their window by one full window, so a request landing
+ * on a boundary cannot read a key that was reclaimed mid-window.
+ */
+const TTL_MULTIPLIER: number = 2;
+
+// How often the "counter unavailable" condition may be logged.
+const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS: number = 60 * 1000;
+
+export default class SubscriberNotificationTestSendRateLimit {
+  /*
+   * Ten test emails a quarter of an hour: someone checking each status
+   * page's email in turn, or trying a template change a few times, is
+   * nowhere near it.
+   */
+  private static readonly config: SubscriberNotificationTestSendRateLimitConfig =
+    {
+      windowSeconds: 15 * 60,
+      perUserLimit: 10,
+    };
+
+  private static counterUnavailableLastLoggedAt: number = 0;
+
+  // The limit in force, for tests and for operators' diagnostics.
+  public static getConfig(): SubscriberNotificationTestSendRateLimitConfig {
+    return { ...SubscriberNotificationTestSendRateLimit.config };
+  }
+
+  /*
+   * The signed-in user the request is billed to. The route sits behind
+   * UserMiddleware.requireUserAuthentication, so there always is one; if
+   * somehow there is not, every such request shares one bucket rather than
+   * each getting its own.
+   */
+  public static resolveUserKey(req: ExpressRequest): string {
+    const userId: ObjectID | undefined = (req as OneUptimeRequest)
+      ?.userAuthorization?.userId;
+
+    if (!userId) {
+      return "anonymous";
+    }
+
+    const key: string = userId.toString().toLowerCase();
+
+    return ObjectID.isValidUUID(key) ? key : "invalid";
+  }
+
+  /*
+   * Count this request against the user and decide. The counter is
+   * incremented on every request, a rejected one included, so a caller who
+   * keeps trying keeps their window pinned rather than getting a fresh
+   * allowance for free.
+   */
+  public static async consume(data: {
+    userKey: string;
+  }): Promise<SubscriberNotificationTestSendRateLimitDecision> {
+    const client: ClientType | null = Redis.getClient();
+
+    if (!client || !Redis.isConnected()) {
+      return {
+        outcome:
+          SubscriberNotificationTestSendRateLimitOutcome.CounterUnavailable,
+      };
+    }
+
+    const config: SubscriberNotificationTestSendRateLimitConfig =
+      SubscriberNotificationTestSendRateLimit.config;
+    const windowMs: number = config.windowSeconds * 1000;
+    const windowIndex: number = Math.floor(Date.now() / windowMs);
+    const counterKey: string = `${KEY_PREFIX}u:${data.userKey}:${windowIndex}`;
+
+    try {
+      const count: number = await client.incr(counterKey);
+
+      if (typeof count !== "number") {
+        throw new Error("Rate limit counter returned a non-numeric value");
+      }
+
+      /*
+       * The expiry is set only by the request that created the key: setting
+       * it on every increment would slide the window for as long as the
+       * requests keep coming.
+       */
+      if (count === 1) {
+        await client.expire(counterKey, config.windowSeconds * TTL_MULTIPLIER);
+      }
+
+      if (count > config.perUserLimit) {
+        return {
+          outcome: SubscriberNotificationTestSendRateLimitOutcome.RateLimited,
+          retryAfterSeconds:
+            SubscriberNotificationTestSendRateLimit.getSecondsUntilWindowEnd(
+              config.windowSeconds,
+            ),
+          isFirstRejectionInWindow: count === config.perUserLimit + 1,
+        };
+      }
+
+      return {
+        outcome: SubscriberNotificationTestSendRateLimitOutcome.Allowed,
+      };
+    } catch (err) {
+      if (
+        SubscriberNotificationTestSendRateLimit.shouldLogCounterUnavailable()
+      ) {
+        logger.warn(
+          `SubscriberNotificationTestSendRateLimit: counter failed for user ${data.userKey}`,
+        );
+        logger.warn(err);
+      }
+
+      return {
+        outcome:
+          SubscriberNotificationTestSendRateLimitOutcome.CounterUnavailable,
+      };
+    }
+  }
+
+  /*
+   * Express middleware. Register after UserMiddleware (the counter is per
+   * user) and before the handler, so a flood is refused before it reads a
+   * single row.
+   */
+  public static getMiddleware(): (
+    req: ExpressRequest,
+    res: ExpressResponse,
+    next: NextFunction,
+  ) => Promise<void> {
+    return async (
+      req: ExpressRequest,
+      res: ExpressResponse,
+      next: NextFunction,
+    ): Promise<void> => {
+      const userKey: string =
+        SubscriberNotificationTestSendRateLimit.resolveUserKey(req);
+
+      const decision: SubscriberNotificationTestSendRateLimitDecision =
+        await SubscriberNotificationTestSendRateLimit.consume({
+          userKey: userKey,
+        });
+
+      if (
+        decision.outcome ===
+        SubscriberNotificationTestSendRateLimitOutcome.RateLimited
+      ) {
+        if (decision.retryAfterSeconds) {
+          SubscriberNotificationTestSendRateLimit.setRetryAfterHeader(
+            res,
+            decision.retryAfterSeconds,
+          );
+        }
+
+        if (decision.isFirstRejectionInWindow) {
+          logger.warn(
+            `SubscriberNotificationTestSendRateLimit: rejected test emails from user ${userKey} for the rest of this window`,
+          );
+        }
+
+        return Response.sendErrorResponse(
+          req,
+          res,
+          new TooManyRequestsException(
+            "You have sent yourself too many test emails. Please try again later.",
+          ),
+        );
+      }
+
+      if (
+        decision.outcome ===
+        SubscriberNotificationTestSendRateLimitOutcome.CounterUnavailable
+      ) {
+        // Fails closed: see the note at the top of this file.
+        if (
+          SubscriberNotificationTestSendRateLimit.shouldLogCounterUnavailable()
+        ) {
+          logger.warn(
+            "SubscriberNotificationTestSendRateLimit: rate limit counter unavailable, refusing test emails",
+          );
+        }
+
+        return Response.sendErrorResponse(
+          req,
+          res,
+          new ServiceUnavailableException(
+            "Test emails cannot be sent right now. Please try again in a few minutes.",
+          ),
+        );
+      }
+
+      return next();
+    };
+  }
+
+  /*
+   * Remaining seconds in the current fixed window, so a refused caller is
+   * told to come back when the window actually rolls. The preview limiter
+   * (SubscriberNotificationPreviewRateLimit) counts in the same fixed
+   * windows, and uses this too.
+   */
+  public static getSecondsUntilWindowEnd(windowSeconds: number): number {
+    const windowMs: number = windowSeconds * 1000;
+    const msIntoWindow: number = Date.now() % windowMs;
+
+    return Math.max(1, Math.ceil((windowMs - msIntoWindow) / 1000));
+  }
+
+  /*
+   * A Redis outage means every request takes the unavailable path; one log
+   * line per interval makes the condition visible without burying the rest.
+   */
+  private static shouldLogCounterUnavailable(): boolean {
+    const now: number = Date.now();
+
+    if (
+      now -
+        SubscriberNotificationTestSendRateLimit.counterUnavailableLastLoggedAt <
+      COUNTER_UNAVAILABLE_LOG_INTERVAL_MS
+    ) {
+      return false;
+    }
+
+    SubscriberNotificationTestSendRateLimit.counterUnavailableLastLoggedAt =
+      now;
+
+    return true;
+  }
+
+  private static setRetryAfterHeader(
+    res: ExpressResponse,
+    retryAfterSeconds: number,
+  ): void {
+    const setHeader: unknown = (res as unknown as Record<string, unknown>)[
+      "setHeader"
+    ];
+
+    if (typeof setHeader === "function") {
+      (setHeader as (name: string, value: string) => void).call(
+        res,
+        "Retry-After",
+        String(retryAfterSeconds),
+      );
+    }
+  }
+}

package/Server/Middleware/TelemetryIngest.ts

@@ -14,6 +14,7 @@ import TelemetryIngestionKeyPolicy, {
 import TelemetryIngestionKeyType from "../../Types/Telemetry/TelemetryIngestionKeyType";
 import TelemetryIngestSurface, {
   BROWSER_ALLOWED_INGEST_SURFACES,
+  IDENTITY_REGISTRATION_SURFACES,
   getIngestSurfaceReadableName,
 } from "../../Types/Telemetry/TelemetryIngestSurface";
 import OriginAllowList from "../../Utils/Telemetry/OriginAllowList";
@@ -72,14 +73,32 @@ const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS: number = 60 * 1000;
 const UNNAMED_SURFACE_READABLE_NAME: string = "this ingest endpoint";
 
 /*
- * The ceiling a server key gets on the Kubernetes agent Runner registration
- * surface when its policy names none. Registration is a handful of calls
- * per agent pod lifetime (start-up, then heartbeats through the Runner's
- * own credential), so a leaked ingestion key hammering it is abuse, not
+ * The ceiling a server key gets on an identity registration surface
+ * (IDENTITY_REGISTRATION_SURFACES: the Kubernetes agent Runner and the
+ * Kubernetes AI agent) when its policy names none. Registration is a handful
+ * of calls per pod lifetime (start-up, then heartbeats through the minted
+ * identity's own key), so a leaked ingestion key hammering it is abuse, not
  * traffic - unlike OTLP ingest, where "no limit" is the historical contract
  * and must stay that way. An explicit per-key limit still wins.
  */
-export const DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE: number = 30;
+export const DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE: number = 30;
+
+// The name this ceiling shipped under, when only the Runner registered.
+export const DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE: number =
+  DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE;
+
+/*
+ * The refusal a service-pinned key gets on an identity registration surface.
+ * Worded for whichever identity the surface mints, and it never names the
+ * key, its pinned service or anything else about it.
+ */
+export const getPinnedKeyRegistrationRefusalMessage: (
+  surface: TelemetryIngestSurface,
+) => string = (surface: TelemetryIngestSurface): string => {
+  return `This telemetry ingestion key is pinned to a single service, so it cannot be used for ${getIngestSurfaceReadableName(
+    surface,
+  )}: that grants access to a Kubernetes cluster, not to one service's telemetry. Use an unpinned server ingestion key for the Kubernetes agent.`;
+};
 
 type GetEffectiveRequestsPerMinuteLimitFunction = (
   policy: TelemetryIngestionKeyPolicy,
@@ -101,11 +120,13 @@ type GetEffectiveRequestsPerMinuteLimitFunction = (
  * with no ceiling is the thing we are trying to stop shipping: "the customer
  * did not configure a limit" cannot be allowed to mean "unlimited" there.
  *
- * The one surface-dependent case is Kubernetes agent Runner registration:
- * a server key with no configured limit is held to a conservative default
- * there (see DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE), because
- * that endpoint mints Runner identities rather than accepting telemetry.
- * Every other surface keeps the server-key contract above untouched.
+ * The one surface-dependent case is identity registration
+ * (IDENTITY_REGISTRATION_SURFACES: the Kubernetes agent Runner and the
+ * Kubernetes AI agent): a server key with no configured limit is held to a
+ * conservative default there (see
+ * DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE), because those
+ * endpoints mint identities rather than accepting telemetry. Every other
+ * surface keeps the server-key contract above untouched.
  */
 export const getEffectiveRequestsPerMinuteLimit: GetEffectiveRequestsPerMinuteLimitFunction =
   (
@@ -129,8 +150,8 @@ export const getEffectiveRequestsPerMinuteLimit: GetEffectiveRequestsPerMinuteLi
       return DEFAULT_BROWSER_KEY_REQUESTS_PER_MINUTE;
     }
 
-    if (surface === TelemetryIngestSurface.KubernetesAgentRunner) {
-      return DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE;
+    if (surface && IDENTITY_REGISTRATION_SURFACES.has(surface)) {
+      return DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE;
     }
 
     return null;
@@ -460,22 +481,24 @@ export default class TelemetryIngest {
       }
 
       /*
-       * Kubernetes agent Runner registration mints a Runner identity — the
-       * one every kubectl job for a cluster is targeted at — so it takes a
-       * project-wide server key. A key pinned to one service's name was
-       * scoped by its owner to that service's telemetry; it is refused here
+       * An identity registration surface (the Kubernetes agent Runner, the
+       * Kubernetes AI agent) mints the identity every kubectl job for a
+       * cluster is targeted at, so it takes a project-wide server key. A key
+       * pinned to one service's name was scoped by its owner to that
+       * service's telemetry; it is refused here, before the rate limiter,
        * rather than letting it stand up cluster access it was never meant
        * to reach.
        */
       if (
-        surface === TelemetryIngestSurface.KubernetesAgentRunner &&
+        surface &&
+        IDENTITY_REGISTRATION_SURFACES.has(surface) &&
         policy.pinnedServiceName
       ) {
         return Response.sendErrorResponse(
           req,
           res,
           new NotAuthorizedException(
-            "This telemetry ingestion key is pinned to a single service, so it cannot register the Kubernetes agent's in-cluster Runner. Use an unpinned server ingestion key for the Kubernetes agent.",
+            getPinnedKeyRegistrationRefusalMessage(surface),
           ),
         );
       }