@oneuptime/common 14.0.6 → 14.0.8

package/Tests/Server/Utils/StatusPage/SubscriberNotificationRunLimit.test.ts

@@ -0,0 +1,275 @@
+import Semaphore, {
+  SemaphoreLockTimeoutError,
+  SemaphorePermit,
+} from "../../../../Server/Infrastructure/Semaphore";
+import SubscriberNotificationRunLimit from "../../../../Server/Utils/StatusPage/SubscriberNotificationRunLimit";
+import SubscriberNotificationTiming from "../../../../Server/Utils/StatusPage/SubscriberNotificationTiming";
+import logger from "../../../../Server/Utils/Logger";
+import { JSONObject } from "../../../../Types/JSON";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  jest,
+  test,
+} from "@jest/globals";
+import type { SpyInstance } from "jest-mock";
+import getJestMockFunction, { MockFunction } from "../../../MockType";
+
+/*
+ * The subscriber jobs run every minute on the shared Worker queue and await
+ * their sends, so their runs overlap; without a bound, slow sends during an
+ * outage could hold most of the Worker consumer's slots and starve on-call
+ * escalation and every other cron. A run takes one of a few permits per job
+ * first (SubscriberNotificationRunLimit), and returns at once when they are
+ * all taken.
+ */
+
+const JOB_NAME: string = "IncidentPublicNote:SendNotificationToSubscribers";
+
+// The nine subscriber jobs, and the Worker consumer's default concurrency.
+const SUBSCRIBER_JOB_COUNT: number = 9;
+const DEFAULT_WORKER_CONCURRENCY: number = 100;
+
+let acquire: SpyInstance<typeof Semaphore.acquirePermit>;
+let release: SpyInstance<typeof Semaphore.releasePermit>;
+let permit: SemaphorePermit;
+
+beforeEach(() => {
+  permit = { identifier: "permit-1" } as unknown as SemaphorePermit;
+
+  acquire = jest
+    .spyOn(Semaphore, "acquirePermit")
+    .mockResolvedValue(permit as never);
+  release = jest
+    .spyOn(Semaphore, "releasePermit")
+    .mockResolvedValue(undefined as never);
+
+  jest.spyOn(logger, "warn").mockImplementation((() => {}) as never);
+  jest.spyOn(logger, "debug").mockImplementation((() => {}) as never);
+});
+
+afterEach(() => {
+  jest.restoreAllMocks();
+});
+
+describe("SubscriberNotificationRunLimit", () => {
+  test("keeps the nine jobs well inside the Worker consumer's slots", () => {
+    expect(
+      SubscriberNotificationRunLimit.MAX_CONCURRENT_RUNS_PER_JOB *
+        SUBSCRIBER_JOB_COUNT,
+    ).toBeLessThanOrEqual(DEFAULT_WORKER_CONCURRENCY / 2);
+    expect(
+      SubscriberNotificationRunLimit.MAX_CONCURRENT_RUNS_PER_JOB,
+    ).toBeGreaterThanOrEqual(2);
+  });
+
+  test("a dead worker's permit lapses long before the job timeout", () => {
+    expect(SubscriberNotificationRunLimit.PERMIT_TTL_IN_MS).toBeLessThan(
+      SubscriberNotificationTiming.JOB_TIMEOUT_IN_MS / 5,
+    );
+  });
+
+  test("runs while holding one of the job's permits, and gives it back after", async () => {
+    const order: Array<string> = [];
+    acquire.mockImplementation((async () => {
+      order.push("acquire");
+      return permit;
+    }) as never);
+    release.mockImplementation((async () => {
+      order.push("release");
+    }) as never);
+
+    await SubscriberNotificationRunLimit.limit(JOB_NAME, async () => {
+      order.push("run");
+    })();
+
+    expect(order).toEqual(["acquire", "run", "release"]);
+    expect(release).toHaveBeenCalledWith(permit);
+
+    const options: JSONObject = acquire.mock.calls[0]![0] as JSONObject;
+    expect(options).toEqual(
+      expect.objectContaining({
+        key: JOB_NAME,
+        namespace: SubscriberNotificationRunLimit.NAMESPACE,
+        limit: SubscriberNotificationRunLimit.MAX_CONCURRENT_RUNS_PER_JOB,
+        lockTimeout: SubscriberNotificationRunLimit.PERMIT_TTL_IN_MS,
+        acquireAttemptsLimit: 1,
+      }),
+    );
+  });
+
+  test("each job has its own permits", async () => {
+    await SubscriberNotificationRunLimit.limit("Job:A", async () => {})();
+    await SubscriberNotificationRunLimit.limit("Job:B", async () => {})();
+
+    expect(
+      acquire.mock.calls.map((call: Array<unknown>): unknown => {
+        return (call[0] as JSONObject)["key"];
+      }),
+    ).toEqual(["Job:A", "Job:B"]);
+  });
+
+  test("when every permit is taken, the run does not run: its notifications wait for the next", async () => {
+    acquire.mockRejectedValue(
+      new SemaphoreLockTimeoutError("Acquire semaphore timeout") as never,
+    );
+    const run: MockFunction = getJestMockFunction();
+
+    await SubscriberNotificationRunLimit.limit(JOB_NAME, async () => {
+      run();
+    })();
+
+    expect(run).not.toHaveBeenCalled();
+    expect(release).not.toHaveBeenCalled();
+  });
+
+  test("gives the permit back when the run throws, and lets the error through", async () => {
+    await expect(
+      SubscriberNotificationRunLimit.limit(JOB_NAME, async () => {
+        throw new Error("database hiccup");
+      })(),
+    ).rejects.toThrow("database hiccup");
+
+    expect(release).toHaveBeenCalledWith(permit);
+  });
+
+  test("runs without a permit when Redis cannot be reached for one", async () => {
+    acquire.mockRejectedValue(
+      new Error("Redis client is not connected") as never,
+    );
+    const run: MockFunction = getJestMockFunction();
+
+    await SubscriberNotificationRunLimit.limit(JOB_NAME, async () => {
+      run();
+    })();
+
+    expect(run).toHaveBeenCalledTimes(1);
+    expect(release).not.toHaveBeenCalled();
+    expect(logger.warn).toHaveBeenCalled();
+  });
+
+  test("a permit that cannot be given back does not fail the run: it lapses", async () => {
+    release.mockRejectedValue(new Error("connection reset") as never);
+
+    await expect(
+      SubscriberNotificationRunLimit.limit(JOB_NAME, async () => {})(),
+    ).resolves.toBeUndefined();
+  });
+
+  test("a permit lost while sending is logged, not thrown from a timer", async () => {
+    await SubscriberNotificationRunLimit.limit(JOB_NAME, async () => {})();
+
+    const onLockLost: (err: Error) => void = (
+      acquire.mock.calls[0]![0] as { onLockLost: (err: Error) => void }
+    ).onLockLost;
+
+    expect(() => {
+      onLockLost(new Error("Lost semaphore for key"));
+    }).not.toThrow();
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.stringContaining("lost its run permit"),
+    );
+  });
+});
+
+/*
+ * The run permits are shared by every project on the server, and a run sends
+ * one notification at a time. So one project's notifications also take one
+ * of a few per-project slots each: a tenant whose sends are slow - its own
+ * mail server hangs, or it posts a burst of notes on a large page - holds at
+ * most that many runs of a job, and the rest keep sending everyone else's.
+ */
+describe("SubscriberNotificationRunLimit.takeProjectSlot", () => {
+  const PROJECT_ID: string = "10000000-0000-4000-8000-00000000000A";
+
+  test("a project gets fewer slots than the job has runs", () => {
+    expect(
+      SubscriberNotificationRunLimit.MAX_CONCURRENT_SENDS_PER_PROJECT,
+    ).toBeGreaterThanOrEqual(1);
+    expect(
+      SubscriberNotificationRunLimit.MAX_CONCURRENT_SENDS_PER_PROJECT,
+    ).toBeLessThan(SubscriberNotificationRunLimit.MAX_CONCURRENT_RUNS_PER_JOB);
+  });
+
+  test("takes one of the project's slots for this job, and gives it back", async () => {
+    const slot: Awaited<
+      ReturnType<typeof SubscriberNotificationRunLimit.takeProjectSlot>
+    > = await SubscriberNotificationRunLimit.takeProjectSlot({
+      jobName: JOB_NAME,
+      projectId: PROJECT_ID,
+    });
+
+    expect(slot).not.toBeNull();
+    expect(acquire.mock.calls[0]![0]).toEqual(
+      expect.objectContaining({
+        // Per job and project; the project id as it is stored.
+        key: `${JOB_NAME}:${PROJECT_ID.toLowerCase()}`,
+        namespace: SubscriberNotificationRunLimit.PROJECT_NAMESPACE,
+        limit: SubscriberNotificationRunLimit.MAX_CONCURRENT_SENDS_PER_PROJECT,
+        lockTimeout: SubscriberNotificationRunLimit.PERMIT_TTL_IN_MS,
+        acquireAttemptsLimit: 1,
+      }),
+    );
+
+    await slot!.release();
+    expect(release).toHaveBeenCalledWith(permit);
+  });
+
+  test("a project that already has its share being sent gets none: its notification waits", async () => {
+    acquire.mockRejectedValue(
+      new SemaphoreLockTimeoutError("Acquire semaphore timeout") as never,
+    );
+
+    await expect(
+      SubscriberNotificationRunLimit.takeProjectSlot({
+        jobName: JOB_NAME,
+        projectId: PROJECT_ID,
+      }),
+    ).resolves.toBeNull();
+  });
+
+  test("sends without a slot when Redis cannot be reached, as a run does without its permit", async () => {
+    acquire.mockRejectedValue(
+      new Error("Redis client is not connected") as never,
+    );
+
+    const slot: Awaited<
+      ReturnType<typeof SubscriberNotificationRunLimit.takeProjectSlot>
+    > = await SubscriberNotificationRunLimit.takeProjectSlot({
+      jobName: JOB_NAME,
+      projectId: PROJECT_ID,
+    });
+
+    expect(slot).not.toBeNull();
+    await slot!.release();
+    expect(release).not.toHaveBeenCalled();
+    expect(logger.warn).toHaveBeenCalled();
+  });
+
+  test("a notification without a project is not bounded by one", async () => {
+    const slot: Awaited<
+      ReturnType<typeof SubscriberNotificationRunLimit.takeProjectSlot>
+    > = await SubscriberNotificationRunLimit.takeProjectSlot({
+      jobName: JOB_NAME,
+      projectId: undefined,
+    });
+
+    expect(slot).not.toBeNull();
+    expect(acquire).not.toHaveBeenCalled();
+  });
+
+  test("a slot that cannot be given back does not throw: it lapses", async () => {
+    release.mockRejectedValue(new Error("connection reset") as never);
+
+    const slot: Awaited<
+      ReturnType<typeof SubscriberNotificationRunLimit.takeProjectSlot>
+    > = await SubscriberNotificationRunLimit.takeProjectSlot({
+      jobName: JOB_NAME,
+      projectId: PROJECT_ID,
+    });
+
+    await expect(slot!.release()).resolves.toBeUndefined();
+  });
+});

package/Tests/Server/Utils/StatusPage/SubscriberNotificationTiming.test.ts

@@ -0,0 +1,131 @@
+import SubscriberNotificationTiming, {
+  SubscriberNotificationRunClock,
+  SubscriberNotificationSendWindow,
+} from "../../../../Server/Utils/StatusPage/SubscriberNotificationTiming";
+import LIMIT_MAX from "../../../../Types/Database/LimitMax";
+import OneUptimeDate from "../../../../Types/Date";
+import { describe, expect, test } from "@jest/globals";
+
+/*
+ * The numbers that bound a subscriber send, and the arithmetic that ties
+ * them together (see SubscriberNotificationTiming). These tests pin the
+ * relations, not just the values: a change to one number that breaks
+ * another's reason fails here rather than in production, as a send that
+ * outlives its job or a live send swept as interrupted.
+ */
+
+const MINUTE: number = OneUptimeDate.convertMinutesToMilliseconds(1);
+
+describe("SubscriberNotificationTiming", () => {
+  test("a bounded number of messages in flight", () => {
+    expect(SubscriberNotificationTiming.SEND_CONCURRENCY).toBe(20);
+  });
+
+  test("reads a status page's subscribers LIMIT_MAX at a time: the cap one read used to stop at", () => {
+    expect(SubscriberNotificationTiming.SUBSCRIBERS_PER_READ).toBe(LIMIT_MAX);
+  });
+
+  test("waits for a message longer than any sender's own retries", () => {
+    // SMTP: three attempts with a 60 second connect timeout, 1 + 2 seconds of backoff and jitter.
+    const smtpRetryLadder: number = 3 * 60_000 + 1_000 + 2_000 + 2 * 1_000;
+    // Slack and webhooks: four attempts, backing off 2, 4 and 8 seconds.
+    const webhookRetryLadder: number = 2_000 + 4_000 + 8_000;
+
+    expect(SubscriberNotificationTiming.SEND_TIMEOUT_IN_MS).toBeGreaterThan(
+      smtpRetryLadder,
+    );
+    expect(SubscriberNotificationTiming.SEND_TIMEOUT_IN_MS).toBeGreaterThan(
+      webhookRetryLadder,
+    );
+  });
+
+  test("the send window fits one full read of a page at a few seconds a message", () => {
+    const rounds: number =
+      SubscriberNotificationTiming.SUBSCRIBERS_PER_READ /
+      SubscriberNotificationTiming.SEND_CONCURRENCY;
+
+    // 10,000 subscribers at 20 in flight: 500 rounds of 2.4 seconds.
+    expect(rounds).toBe(500);
+    expect(
+      SubscriberNotificationTiming.SEND_WINDOW_IN_MS / rounds,
+    ).toBeGreaterThanOrEqual(2_000);
+  });
+
+  test("a notification at most takes its window, the last answers it waits for, and its settle", () => {
+    expect(SubscriberNotificationTiming.NOTIFICATION_MAX_IN_MS).toBe(
+      SubscriberNotificationTiming.SEND_WINDOW_IN_MS +
+        SubscriberNotificationTiming.SEND_TIMEOUT_IN_MS +
+        SubscriberNotificationTiming.SETTLE_ALLOWANCE_IN_MS,
+    );
+    expect(SubscriberNotificationTiming.NOTIFICATION_MAX_IN_MS).toBe(
+      25 * MINUTE,
+    );
+  });
+
+  test("a run claims only what can finish within the job timeout, and there is time to claim", () => {
+    expect(SubscriberNotificationTiming.JOB_TIMEOUT_IN_MS).toBe(30 * MINUTE);
+    expect(
+      SubscriberNotificationTiming.LATEST_CLAIM_IN_MS +
+        SubscriberNotificationTiming.NOTIFICATION_MAX_IN_MS,
+    ).toBe(SubscriberNotificationTiming.JOB_TIMEOUT_IN_MS);
+    expect(SubscriberNotificationTiming.LATEST_CLAIM_IN_MS).toBeGreaterThan(
+      MINUTE,
+    );
+  });
+
+  test("the sweeper waits well past anything a live send can take", () => {
+    expect(SubscriberNotificationTiming.STUCK_AFTER_IN_MS).toBe(40 * MINUTE);
+    expect(
+      SubscriberNotificationTiming.STUCK_AFTER_IN_MS -
+        SubscriberNotificationTiming.JOB_TIMEOUT_IN_MS,
+    ).toBeGreaterThanOrEqual(5 * MINUTE);
+  });
+
+  describe("the run clock", () => {
+    test("claims until the latest claim, and not after", () => {
+      const clock: SubscriberNotificationRunClock =
+        SubscriberNotificationTiming.startRun(1_000_000);
+
+      expect(clock.canClaimAnotherNotification(1_000_000)).toBe(true);
+      expect(
+        clock.canClaimAnotherNotification(
+          1_000_000 + SubscriberNotificationTiming.LATEST_CLAIM_IN_MS,
+        ),
+      ).toBe(true);
+      expect(
+        clock.canClaimAnotherNotification(
+          1_000_000 + SubscriberNotificationTiming.LATEST_CLAIM_IN_MS + 1,
+        ),
+      ).toBe(false);
+    });
+
+    test("reads the time now by default", () => {
+      expect(
+        SubscriberNotificationTiming.startRun().canClaimAnotherNotification(),
+      ).toBe(true);
+    });
+  });
+
+  describe("the send window", () => {
+    test("is open for SEND_WINDOW_IN_MS after the claim", () => {
+      const window: SubscriberNotificationSendWindow =
+        SubscriberNotificationTiming.startSendWindow(5_000);
+
+      expect(window.closesAt).toBe(
+        5_000 + SubscriberNotificationTiming.SEND_WINDOW_IN_MS,
+      );
+      expect(window.isOpen(5_000)).toBe(true);
+      expect(window.isOpen(window.closesAt - 1)).toBe(true);
+      expect(window.isOpen(window.closesAt)).toBe(false);
+    });
+
+    test("reads the time now by default", () => {
+      expect(SubscriberNotificationTiming.startSendWindow().isOpen()).toBe(
+        true,
+      );
+      expect(
+        new SubscriberNotificationSendWindow(Date.now() - 1).isOpen(),
+      ).toBe(false);
+    });
+  });
+});

package/Tests/Server/Utils/StatusPageResource.test.ts

@@ -135,6 +135,104 @@ describe("StatusPageResourceUtil", () => {
     });
   });
 
+  /*
+   * The HTML list goes into the raw-HTML slot of the default subscriber
+   * emails, and into custom email templates as HTML, so every name in it is
+   * escaped. Project members name resources and groups; a name holding a
+   * link or a script must read as those characters, not become markup.
+   */
+  describe("getResourcesGroupedByGroupName escapes names", () => {
+    const LINK: string = '<a href="https://evil.example/login">Checkout</a>';
+    const LINK_ESCAPED: string =
+      "&lt;a href=&quot;https://evil.example/login&quot;&gt;Checkout&lt;/a&gt;";
+    const SCRIPT: string = "<script>alert('x')</script>";
+    const SCRIPT_ESCAPED: string =
+      "&lt;script&gt;alert(&#39;x&#39;)&lt;/script&gt;";
+
+    it("escapes an ungrouped display name", () => {
+      const result: string =
+        StatusPageResourceUtil.getResourcesGroupedByGroupName([
+          createResource(LINK),
+          createResource("Payments & Billing"),
+        ]);
+
+      expect(result).toBe(`${LINK_ESCAPED}, Payments &amp; Billing`);
+      expect(result).not.toContain("<a");
+    });
+
+    it("escapes a group name and the names in it", () => {
+      const result: string =
+        StatusPageResourceUtil.getResourcesGroupedByGroupName([
+          createResourceWithGroup(SCRIPT, `EU "West"`),
+          createResourceWithGroup("API", `EU "West"`),
+          createResourceWithGroup("Web", SCRIPT),
+        ]);
+
+      expect(result).toBe(
+        `EU &quot;West&quot;: ${SCRIPT_ESCAPED}, API<br/>${SCRIPT_ESCAPED}: Web`,
+      );
+      expect(result).not.toContain("<script");
+    });
+
+    it("escapes an ungrouped name listed after the groups", () => {
+      const result: string =
+        StatusPageResourceUtil.getResourcesGroupedByGroupName([
+          createResourceWithGroup("API", "EU"),
+          createResource(`O'Brien's <b>service</b>`),
+        ]);
+
+      expect(result).toBe(
+        "EU: API<br/>O&#39;Brien&#39;s &lt;b&gt;service&lt;/b&gt;",
+      );
+    });
+
+    it("keeps its own <br/> between groups as markup", () => {
+      const result: string =
+        StatusPageResourceUtil.getResourcesGroupedByGroupName([
+          createResourceWithGroup("A", "G1"),
+          createResourceWithGroup("B", "G2"),
+        ]);
+
+      expect(result).toBe("G1: A<br/>G2: B");
+    });
+
+    it("merges two groups only when their names are the same as written", () => {
+      const result: string =
+        StatusPageResourceUtil.getResourcesGroupedByGroupName([
+          createResourceWithGroup("A", "R&D"),
+          createResourceWithGroup("B", "R&amp;D"),
+          createResourceWithGroup("C", "R&D"),
+        ]);
+
+      expect(result).toBe("R&amp;D: A, C<br/>R&amp;amp;D: B");
+    });
+
+    it("escapes the default value too", () => {
+      expect(
+        StatusPageResourceUtil.getResourcesGroupedByGroupName([], "<none>"),
+      ).toBe("&lt;none&gt;");
+    });
+
+    it("leaves the plain-text list as written, for channels that do not render HTML", () => {
+      const resources: Array<StatusPageResource> = [
+        createResourceWithGroup(SCRIPT, `EU "West"`),
+        createResource(LINK),
+      ];
+
+      expect(
+        StatusPageResourceUtil.getResourcesGroupedByGroupNameAsPlainText(
+          resources,
+        ),
+      ).toBe(`EU "West": ${SCRIPT}; ${LINK}`);
+      expect(
+        StatusPageResourceUtil.getResourcesGroupedByGroupNameAsPlainText(
+          [],
+          "<none>",
+        ),
+      ).toBe("<none>");
+    });
+  });
+
   describe("getResourcesGroupedByGroupNameAsPlainText", () => {
     it("should separate groups with semicolons instead of <br/>", () => {
       const resources: Array<StatusPageResource> = [