@oneuptime/common 14.0.7 → 14.0.8

package/Server/Middleware/IdentityRateLimit.ts

@@ -74,6 +74,17 @@ import TooManyRequestsException from "../../Types/Exception/TooManyRequestsExcep
  * DatabaseNotConnectedException when Redis is down. Logging in is already
  * impossible in that state; the difference this makes is a 503 that says to
  * come back shortly instead of a 500.
+ *
+ * ONE ROUTE HERE SENDS MAIL RATHER THAN CHECKING A CREDENTIAL
+ *
+ * POST /resend-verification-email (Authentication.ts) is limited by this
+ * middleware too, on a bucket of its own. It is not a guessing oracle against
+ * a stored credential -- the credential it takes is server-minted and signed,
+ * or a random UUID from an emailed link -- but every request that gets past its
+ * checks makes the server send an email, and an anonymous route that sends
+ * mail needs a ceiling in front of its database reads and its caps. It shares
+ * the fail-closed choice for the same reason: its own cooldown fence lives in
+ * Redis, so with Redis down it could not bound anything either.
  */
 
 /*
@@ -144,6 +155,15 @@ export enum IdentityRateLimitBucket {
 
   // Discoverable passkey requests have no account identifier until verified.
   Passkey = "passkey",
+
+  /*
+   * POST /resend-verification-email -- an unverified account asking for its
+   * verification link again. Mail, not guessing: see
+   * VERIFICATION_EMAIL_RESEND_BUCKET. Its own counter so that a user mashing
+   * "resend" cannot spend the sign-in attempts they are about to need, and so
+   * that sign-in noise cannot refuse the resend.
+   */
+  VerificationEmailResend = "verification-email-resend",
 }
 
 export enum IdentityRateLimitScope {
@@ -413,6 +433,55 @@ const PASSKEY_BUCKET: BucketConfig = {
   perIpLimit: PASSKEY_RATE_LIMIT,
 };
 
+/*
+ * Verification email resend budget.
+ *
+ * Shaped like the passkey bucket, and for the same reason: the request carries
+ * no email address (it carries a signed resend token or a verification link
+ * token instead), so resolveAccountKey answers "none" and the account counter
+ * is keyed on the client address alone. Both counters are therefore the same
+ * per-address budget and are given the same number; a caller who adds an
+ * `email` field to rotate the account key still meets the address ceiling.
+ *
+ * This is NOT what bounds mail to any one inbox. The route's own caps do that
+ * per account -- a cooldown between sends, a per-hour cap, and a small fixed
+ * number of sends per credential (VerificationEmailResendPolicy) -- and those
+ * hold however many addresses a caller has. What this bounds is how much work
+ * one address can make the route do before those caps are even consulted:
+ * token verification, a verification-token lookup, a user lookup and a Redis
+ * read per request.
+ *
+ * Twenty a quarter hour is far past an honest user, who presses the button a
+ * handful of times at most because the page itself counts the cooldown down,
+ * and still generous to an office behind one NAT where several people are
+ * finishing signup at once.
+ */
+const VERIFICATION_EMAIL_RESEND_RATE_LIMIT: number = parsePositiveIntFromEnv(
+  "IDENTITY_VERIFICATION_EMAIL_RESEND_RATE_LIMIT_PER_IP_PER_WINDOW",
+  20,
+);
+const VERIFICATION_EMAIL_RESEND_BUCKET: BucketConfig = {
+  windowSeconds: parsePositiveIntFromEnv(
+    "IDENTITY_VERIFICATION_EMAIL_RESEND_RATE_LIMIT_WINDOW_SECONDS",
+    15 * 60,
+  ),
+  perAccountLimit: VERIFICATION_EMAIL_RESEND_RATE_LIMIT,
+  perIpLimit: VERIFICATION_EMAIL_RESEND_RATE_LIMIT,
+};
+
+/*
+ * What a refused caller is told. See getRefusalMessages for why the sign-in
+ * routes all share one pair and the resend route has its own.
+ */
+export const IDENTITY_RATE_LIMITED_MESSAGE: string =
+  "Too many sign-in attempts. Please try again later.";
+export const IDENTITY_COUNTER_UNAVAILABLE_MESSAGE: string =
+  "Unable to sign you in right now. Please try again shortly.";
+export const VERIFICATION_EMAIL_RESEND_RATE_LIMITED_MESSAGE: string =
+  "Too many requests for a new verification email. Please try again later.";
+export const VERIFICATION_EMAIL_RESEND_UNAVAILABLE_MESSAGE: string =
+  "Unable to send a verification email right now. Please try again shortly.";
+
 const KEY_PREFIX: string = "identity:rl:";
 
 /*
@@ -574,9 +643,44 @@ export default class IdentityRateLimit {
       return STATUS_PAGE_LOGIN_BUCKET;
     }
 
+    if (bucket === IdentityRateLimitBucket.VerificationEmailResend) {
+      return VERIFICATION_EMAIL_RESEND_BUCKET;
+    }
+
     return LOGIN_BUCKET;
   }
 
+  /*
+   * What a refused request is told, for the 429 and for the fail-closed 503.
+   *
+   * Every sign-in bucket gets the same pair, byte for byte, whichever counter
+   * fired and whichever route was hit -- see the note in getMiddleware on why
+   * a sign-in limiter must not answer differently for different accounts.
+   *
+   * The resend route gets its own pair because the sign-in wording is simply
+   * wrong there: somebody who pressed "resend verification email" has made no
+   * sign-in attempt, and telling them they made too many would send them
+   * looking for a problem they do not have. The pair is still uniform WITHIN
+   * the route: the same text for every caller, token and account, so it
+   * discloses nothing the sign-in pair does not.
+   */
+  public static getRefusalMessages(bucket: IdentityRateLimitBucket): {
+    rateLimited: string;
+    unavailable: string;
+  } {
+    if (bucket === IdentityRateLimitBucket.VerificationEmailResend) {
+      return {
+        rateLimited: VERIFICATION_EMAIL_RESEND_RATE_LIMITED_MESSAGE,
+        unavailable: VERIFICATION_EMAIL_RESEND_UNAVAILABLE_MESSAGE,
+      };
+    }
+
+    return {
+      rateLimited: IDENTITY_RATE_LIMITED_MESSAGE,
+      unavailable: IDENTITY_COUNTER_UNAVAILABLE_MESSAGE,
+    };
+  }
+
   /*
    * Increment both counters for this request and decide.
    *
@@ -773,13 +877,14 @@ export default class IdentityRateLimit {
          * account exists. These routes are careful not to say which half of a
          * credential was wrong; a limiter that answered differently for a real
          * address than for an invented one would hand back the account
-         * enumeration the handlers withhold.
+         * enumeration the handlers withhold. (The resend route has wording of
+         * its own, but it is just as uniform; see getRefusalMessages.)
          */
         return Response.sendErrorResponse(
           req,
           res,
           new TooManyRequestsException(
-            "Too many sign-in attempts. Please try again later.",
+            IdentityRateLimit.getRefusalMessages(bucket).rateLimited,
           ),
         );
       }
@@ -800,7 +905,7 @@ export default class IdentityRateLimit {
           req,
           res,
           new ServiceUnavailableException(
-            "Unable to sign you in right now. Please try again shortly.",
+            IdentityRateLimit.getRefusalMessages(bucket).unavailable,
           ),
         );
       }

package/Server/Middleware/SubscriberNotificationPreviewRateLimit.ts

@@ -0,0 +1,237 @@
+import Redis, { ClientType } from "../Infrastructure/Redis";
+import {
+  ExpressRequest,
+  ExpressResponse,
+  NextFunction,
+} from "../Utils/Express";
+import logger from "../Utils/Logger";
+import Response from "../Utils/Response";
+import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
+import SubscriberNotificationTestSendRateLimit from "./SubscriberNotificationTestSendRateLimit";
+
+/*
+ * Request limiting for "Preview notification" (POST /preview on the
+ * subscriber notification preview router).
+ *
+ * A preview sends nothing, but it is not free: it renders the incident's
+ * description and custom fields once, then an email for every status page
+ * the caller can read, and returns all of that HTML in one answer. Any
+ * project member holding one of the audience roles may call it, as often as
+ * they like, so this bounds how much rendering one person can ask the App
+ * server for. The request itself is bounded too
+ * (SubscriberNotificationPreviewBuilder.parseRequest caps the text and the
+ * custom field values).
+ *
+ * The limit is loose: someone writing an incident or a note and previewing
+ * it after each change is nowhere near it. The counter is per user, across
+ * every project and incident, like the test email's
+ * (SubscriberNotificationTestSendRateLimit, whose user key this shares).
+ *
+ * THIS FAILS OPEN
+ *
+ * If Redis is unreachable the preview is served. That is the opposite of the
+ * test email's choice, and the difference is what is at stake: a test email
+ * is real mail, and this counter would be the only bound on it; a preview
+ * sends nothing, its request is size-capped, and refusing it would take the
+ * preview away from everyone for as long as Redis is down.
+ */
+
+export enum SubscriberNotificationPreviewRateLimitOutcome {
+  Allowed = "allowed",
+  RateLimited = "rate-limited",
+
+  // Redis is unreachable: served, since this fails open.
+  CounterUnavailable = "counter-unavailable",
+}
+
+export interface SubscriberNotificationPreviewRateLimitDecision {
+  outcome: SubscriberNotificationPreviewRateLimitOutcome;
+  retryAfterSeconds?: number | undefined;
+  // True only on the request that first crossed the line in this window.
+  isFirstRejectionInWindow?: boolean | undefined;
+}
+
+export interface SubscriberNotificationPreviewRateLimitConfig {
+  windowSeconds: number;
+  perUserLimit: number;
+}
+
+const KEY_PREFIX: string = "subnotifpreview:rl:";
+
+// Counter keys outlive their window by one full window.
+const TTL_MULTIPLIER: number = 2;
+
+// How often the "counter unavailable" condition may be logged.
+const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS: number = 60 * 1000;
+
+export default class SubscriberNotificationPreviewRateLimit {
+  // Sixty previews every ten minutes: one every ten seconds, sustained.
+  private static readonly config: SubscriberNotificationPreviewRateLimitConfig =
+    {
+      windowSeconds: 10 * 60,
+      perUserLimit: 60,
+    };
+
+  private static counterUnavailableLastLoggedAt: number = 0;
+
+  // The limit in force, for tests and for operators' diagnostics.
+  public static getConfig(): SubscriberNotificationPreviewRateLimitConfig {
+    return { ...SubscriberNotificationPreviewRateLimit.config };
+  }
+
+  /*
+   * Count this request against the user and decide. A rejected request is
+   * counted too, so a caller who keeps trying keeps their window pinned.
+   */
+  public static async consume(data: {
+    userKey: string;
+  }): Promise<SubscriberNotificationPreviewRateLimitDecision> {
+    const client: ClientType | null = Redis.getClient();
+
+    if (!client || !Redis.isConnected()) {
+      return {
+        outcome:
+          SubscriberNotificationPreviewRateLimitOutcome.CounterUnavailable,
+      };
+    }
+
+    const config: SubscriberNotificationPreviewRateLimitConfig =
+      SubscriberNotificationPreviewRateLimit.config;
+    const windowMs: number = config.windowSeconds * 1000;
+    const windowIndex: number = Math.floor(Date.now() / windowMs);
+    const counterKey: string = `${KEY_PREFIX}u:${data.userKey}:${windowIndex}`;
+
+    try {
+      const count: number = await client.incr(counterKey);
+
+      if (typeof count !== "number") {
+        throw new Error("Rate limit counter returned a non-numeric value");
+      }
+
+      // Set only by the request that created the key, so the window is fixed.
+      if (count === 1) {
+        await client.expire(counterKey, config.windowSeconds * TTL_MULTIPLIER);
+      }
+
+      if (count > config.perUserLimit) {
+        return {
+          outcome: SubscriberNotificationPreviewRateLimitOutcome.RateLimited,
+          retryAfterSeconds:
+            SubscriberNotificationTestSendRateLimit.getSecondsUntilWindowEnd(
+              config.windowSeconds,
+            ),
+          isFirstRejectionInWindow: count === config.perUserLimit + 1,
+        };
+      }
+
+      return {
+        outcome: SubscriberNotificationPreviewRateLimitOutcome.Allowed,
+      };
+    } catch (err) {
+      if (
+        SubscriberNotificationPreviewRateLimit.shouldLogCounterUnavailable()
+      ) {
+        logger.warn(
+          `SubscriberNotificationPreviewRateLimit: counter failed for user ${data.userKey}`,
+        );
+        logger.warn(err);
+      }
+
+      return {
+        outcome:
+          SubscriberNotificationPreviewRateLimitOutcome.CounterUnavailable,
+      };
+    }
+  }
+
+  /*
+   * Express middleware. Register after UserMiddleware (the counter is per
+   * user) and before the handler, so a flood is refused before it renders
+   * anything.
+   */
+  public static getMiddleware(): (
+    req: ExpressRequest,
+    res: ExpressResponse,
+    next: NextFunction,
+  ) => Promise<void> {
+    return async (
+      req: ExpressRequest,
+      res: ExpressResponse,
+      next: NextFunction,
+    ): Promise<void> => {
+      const userKey: string =
+        SubscriberNotificationTestSendRateLimit.resolveUserKey(req);
+
+      const decision: SubscriberNotificationPreviewRateLimitDecision =
+        await SubscriberNotificationPreviewRateLimit.consume({
+          userKey: userKey,
+        });
+
+      if (
+        decision.outcome ===
+        SubscriberNotificationPreviewRateLimitOutcome.RateLimited
+      ) {
+        if (decision.retryAfterSeconds) {
+          const setHeader: unknown = (
+            res as unknown as Record<string, unknown>
+          )["setHeader"];
+
+          if (typeof setHeader === "function") {
+            (setHeader as (name: string, value: string) => void).call(
+              res,
+              "Retry-After",
+              String(decision.retryAfterSeconds),
+            );
+          }
+        }
+
+        if (decision.isFirstRejectionInWindow) {
+          logger.warn(
+            `SubscriberNotificationPreviewRateLimit: rejected previews from user ${userKey} for the rest of this window`,
+          );
+        }
+
+        return Response.sendErrorResponse(
+          req,
+          res,
+          new TooManyRequestsException(
+            "You have previewed notifications too many times in a short while. Please try again in a few minutes.",
+          ),
+        );
+      }
+
+      if (
+        decision.outcome ===
+          SubscriberNotificationPreviewRateLimitOutcome.CounterUnavailable &&
+        SubscriberNotificationPreviewRateLimit.shouldLogCounterUnavailable()
+      ) {
+        // Fails open: see the note at the top of this file.
+        logger.warn(
+          "SubscriberNotificationPreviewRateLimit: rate limit counter unavailable, serving previews without a limit",
+        );
+      }
+
+      return next();
+    };
+  }
+
+  /*
+   * A Redis outage means every request takes the unavailable path; one log
+   * line per interval makes the condition visible without burying the rest.
+   */
+  private static shouldLogCounterUnavailable(): boolean {
+    const now: number = Date.now();
+
+    if (
+      now -
+        SubscriberNotificationPreviewRateLimit.counterUnavailableLastLoggedAt <
+      COUNTER_UNAVAILABLE_LOG_INTERVAL_MS
+    ) {
+      return false;
+    }
+
+    SubscriberNotificationPreviewRateLimit.counterUnavailableLastLoggedAt = now;
+
+    return true;
+  }
+}

package/Server/Middleware/SubscriberNotificationTestSendRateLimit.ts

@@ -0,0 +1,304 @@
+import Redis, { ClientType } from "../Infrastructure/Redis";
+import {
+  ExpressRequest,
+  ExpressResponse,
+  NextFunction,
+  OneUptimeRequest,
+} from "../Utils/Express";
+import logger from "../Utils/Logger";
+import Response from "../Utils/Response";
+import ObjectID from "../../Types/ObjectID";
+import ServiceUnavailableException from "../../Types/Exception/ServiceUnavailableException";
+import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
+
+/*
+ * Request limiting for "Send test to me" (POST /send-test on the subscriber
+ * notification preview router), which emails the signed-in user the status
+ * page subscriber email they are previewing.
+ *
+ * The route only ever sends to the caller's own verified account email, so
+ * what this bounds is volume: every allowed request is a real email, through
+ * the status page's own SMTP server or the instance's mail provider, and
+ * nothing else stops a script (or a held-down button) from sending hundreds.
+ * The counter is per user, across every project, status page and incident:
+ * a user is one inbox.
+ *
+ * THIS FAILS CLOSED
+ *
+ * If Redis is unreachable, the test is refused with "try again later". That
+ * is the opposite of the choice VerificationCodeRateLimit makes, and the
+ * difference is what else bounds the requests: there, the code's expiry, its
+ * attempt counter and the resend cooldown all live in Postgres and stay in
+ * force; here this counter is the only bound on how much mail is sent. A
+ * test email is a convenience, and a Redis blip costs nothing but a retry.
+ */
+
+export enum SubscriberNotificationTestSendRateLimitOutcome {
+  Allowed = "allowed",
+  RateLimited = "rate-limited",
+
+  // Redis is unreachable, so the limit cannot be honoured.
+  CounterUnavailable = "counter-unavailable",
+}
+
+export interface SubscriberNotificationTestSendRateLimitDecision {
+  outcome: SubscriberNotificationTestSendRateLimitOutcome;
+  retryAfterSeconds?: number | undefined;
+  /*
+   * True only on the request that first crossed the line in this window, so
+   * a flood produces one log line per user per window.
+   */
+  isFirstRejectionInWindow?: boolean | undefined;
+}
+
+export interface SubscriberNotificationTestSendRateLimitConfig {
+  windowSeconds: number;
+  perUserLimit: number;
+}
+
+const KEY_PREFIX: string = "subnotiftest:rl:";
+
+/*
+ * Counter keys outlive their window by one full window, so a request landing
+ * on a boundary cannot read a key that was reclaimed mid-window.
+ */
+const TTL_MULTIPLIER: number = 2;
+
+// How often the "counter unavailable" condition may be logged.
+const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS: number = 60 * 1000;
+
+export default class SubscriberNotificationTestSendRateLimit {
+  /*
+   * Ten test emails a quarter of an hour: someone checking each status
+   * page's email in turn, or trying a template change a few times, is
+   * nowhere near it.
+   */
+  private static readonly config: SubscriberNotificationTestSendRateLimitConfig =
+    {
+      windowSeconds: 15 * 60,
+      perUserLimit: 10,
+    };
+
+  private static counterUnavailableLastLoggedAt: number = 0;
+
+  // The limit in force, for tests and for operators' diagnostics.
+  public static getConfig(): SubscriberNotificationTestSendRateLimitConfig {
+    return { ...SubscriberNotificationTestSendRateLimit.config };
+  }
+
+  /*
+   * The signed-in user the request is billed to. The route sits behind
+   * UserMiddleware.requireUserAuthentication, so there always is one; if
+   * somehow there is not, every such request shares one bucket rather than
+   * each getting its own.
+   */
+  public static resolveUserKey(req: ExpressRequest): string {
+    const userId: ObjectID | undefined = (req as OneUptimeRequest)
+      ?.userAuthorization?.userId;
+
+    if (!userId) {
+      return "anonymous";
+    }
+
+    const key: string = userId.toString().toLowerCase();
+
+    return ObjectID.isValidUUID(key) ? key : "invalid";
+  }
+
+  /*
+   * Count this request against the user and decide. The counter is
+   * incremented on every request, a rejected one included, so a caller who
+   * keeps trying keeps their window pinned rather than getting a fresh
+   * allowance for free.
+   */
+  public static async consume(data: {
+    userKey: string;
+  }): Promise<SubscriberNotificationTestSendRateLimitDecision> {
+    const client: ClientType | null = Redis.getClient();
+
+    if (!client || !Redis.isConnected()) {
+      return {
+        outcome:
+          SubscriberNotificationTestSendRateLimitOutcome.CounterUnavailable,
+      };
+    }
+
+    const config: SubscriberNotificationTestSendRateLimitConfig =
+      SubscriberNotificationTestSendRateLimit.config;
+    const windowMs: number = config.windowSeconds * 1000;
+    const windowIndex: number = Math.floor(Date.now() / windowMs);
+    const counterKey: string = `${KEY_PREFIX}u:${data.userKey}:${windowIndex}`;
+
+    try {
+      const count: number = await client.incr(counterKey);
+
+      if (typeof count !== "number") {
+        throw new Error("Rate limit counter returned a non-numeric value");
+      }
+
+      /*
+       * The expiry is set only by the request that created the key: setting
+       * it on every increment would slide the window for as long as the
+       * requests keep coming.
+       */
+      if (count === 1) {
+        await client.expire(counterKey, config.windowSeconds * TTL_MULTIPLIER);
+      }
+
+      if (count > config.perUserLimit) {
+        return {
+          outcome: SubscriberNotificationTestSendRateLimitOutcome.RateLimited,
+          retryAfterSeconds:
+            SubscriberNotificationTestSendRateLimit.getSecondsUntilWindowEnd(
+              config.windowSeconds,
+            ),
+          isFirstRejectionInWindow: count === config.perUserLimit + 1,
+        };
+      }
+
+      return {
+        outcome: SubscriberNotificationTestSendRateLimitOutcome.Allowed,
+      };
+    } catch (err) {
+      if (
+        SubscriberNotificationTestSendRateLimit.shouldLogCounterUnavailable()
+      ) {
+        logger.warn(
+          `SubscriberNotificationTestSendRateLimit: counter failed for user ${data.userKey}`,
+        );
+        logger.warn(err);
+      }
+
+      return {
+        outcome:
+          SubscriberNotificationTestSendRateLimitOutcome.CounterUnavailable,
+      };
+    }
+  }
+
+  /*
+   * Express middleware. Register after UserMiddleware (the counter is per
+   * user) and before the handler, so a flood is refused before it reads a
+   * single row.
+   */
+  public static getMiddleware(): (
+    req: ExpressRequest,
+    res: ExpressResponse,
+    next: NextFunction,
+  ) => Promise<void> {
+    return async (
+      req: ExpressRequest,
+      res: ExpressResponse,
+      next: NextFunction,
+    ): Promise<void> => {
+      const userKey: string =
+        SubscriberNotificationTestSendRateLimit.resolveUserKey(req);
+
+      const decision: SubscriberNotificationTestSendRateLimitDecision =
+        await SubscriberNotificationTestSendRateLimit.consume({
+          userKey: userKey,
+        });
+
+      if (
+        decision.outcome ===
+        SubscriberNotificationTestSendRateLimitOutcome.RateLimited
+      ) {
+        if (decision.retryAfterSeconds) {
+          SubscriberNotificationTestSendRateLimit.setRetryAfterHeader(
+            res,
+            decision.retryAfterSeconds,
+          );
+        }
+
+        if (decision.isFirstRejectionInWindow) {
+          logger.warn(
+            `SubscriberNotificationTestSendRateLimit: rejected test emails from user ${userKey} for the rest of this window`,
+          );
+        }
+
+        return Response.sendErrorResponse(
+          req,
+          res,
+          new TooManyRequestsException(
+            "You have sent yourself too many test emails. Please try again later.",
+          ),
+        );
+      }
+
+      if (
+        decision.outcome ===
+        SubscriberNotificationTestSendRateLimitOutcome.CounterUnavailable
+      ) {
+        // Fails closed: see the note at the top of this file.
+        if (
+          SubscriberNotificationTestSendRateLimit.shouldLogCounterUnavailable()
+        ) {
+          logger.warn(
+            "SubscriberNotificationTestSendRateLimit: rate limit counter unavailable, refusing test emails",
+          );
+        }
+
+        return Response.sendErrorResponse(
+          req,
+          res,
+          new ServiceUnavailableException(
+            "Test emails cannot be sent right now. Please try again in a few minutes.",
+          ),
+        );
+      }
+
+      return next();
+    };
+  }
+
+  /*
+   * Remaining seconds in the current fixed window, so a refused caller is
+   * told to come back when the window actually rolls. The preview limiter
+   * (SubscriberNotificationPreviewRateLimit) counts in the same fixed
+   * windows, and uses this too.
+   */
+  public static getSecondsUntilWindowEnd(windowSeconds: number): number {
+    const windowMs: number = windowSeconds * 1000;
+    const msIntoWindow: number = Date.now() % windowMs;
+
+    return Math.max(1, Math.ceil((windowMs - msIntoWindow) / 1000));
+  }
+
+  /*
+   * A Redis outage means every request takes the unavailable path; one log
+   * line per interval makes the condition visible without burying the rest.
+   */
+  private static shouldLogCounterUnavailable(): boolean {
+    const now: number = Date.now();
+
+    if (
+      now -
+        SubscriberNotificationTestSendRateLimit.counterUnavailableLastLoggedAt <
+      COUNTER_UNAVAILABLE_LOG_INTERVAL_MS
+    ) {
+      return false;
+    }
+
+    SubscriberNotificationTestSendRateLimit.counterUnavailableLastLoggedAt =
+      now;
+
+    return true;
+  }
+
+  private static setRetryAfterHeader(
+    res: ExpressResponse,
+    retryAfterSeconds: number,
+  ): void {
+    const setHeader: unknown = (res as unknown as Record<string, unknown>)[
+      "setHeader"
+    ];
+
+    if (typeof setHeader === "function") {
+      (setHeader as (name: string, value: string) => void).call(
+        res,
+        "Retry-After",
+        String(retryAfterSeconds),
+      );
+    }
+  }
+}