@oneuptime/common 14.0.13 → 14.0.14

package/Tests/Server/Utils/Greenlock/CertificateOrderBudget.test.ts

@@ -0,0 +1,243 @@
+/**
+ * CertificateOrderBudget - the Let's Encrypt allowance of the whole
+ * installation, which every certificate order draws from.
+ *
+ * Every certificate is ordered from one Let's Encrypt account, which may
+ * place 300 new orders per three hours, renewals included. The status page
+ * and dashboard sweeps, Check now, the order APIs, reissues, the renewal
+ * runs and CoreSSL each had caps of their own, and with backlogs those caps
+ * added up to more than 400 orders in three hours. This pins the one budget
+ * that bounds them together, and that renewals come first.
+ */
+
+import CertificateOrderBudget, {
+  CertificateOrderReason,
+} from "../../../../Server/Utils/Greenlock/CertificateOrderBudget";
+import GlobalCache from "../../../../Server/Infrastructure/GlobalCache";
+import OneUptimeDate from "../../../../Types/Date";
+import { InMemoryRedis, useInMemoryRedis } from "./InMemoryRedis";
+import { afterEach, describe, expect, jest, test } from "@jest/globals";
+
+const NOW: Date = new Date("2026-10-03T12:07:00.000Z");
+
+const WINDOW_IN_MS: number = OneUptimeDate.convertMinutesToMilliseconds(
+  CertificateOrderBudget.WINDOW_IN_MINUTES,
+);
+
+async function take(
+  reason: CertificateOrderReason,
+  now: Date = NOW,
+): Promise<boolean> {
+  return await CertificateOrderBudget.takeSlot({ reason: reason, now: now });
+}
+
+describe("CertificateOrderBudget limits", () => {
+  /*
+   * Any three hours overlap at most 13 windows of 15 minutes - 12 whole
+   * ones and parts of two more - so that many full windows must still fit
+   * under Let's Encrypt's limit.
+   */
+  test("any three hours stay under Let's Encrypt's 300 new orders per account", () => {
+    const threeHoursInMinutes: number = 3 * 60;
+
+    const windowsInThreeHours: number =
+      Math.ceil(
+        threeHoursInMinutes / CertificateOrderBudget.WINDOW_IN_MINUTES,
+      ) + 1;
+
+    expect(windowsInThreeHours * CertificateOrderBudget.ORDERS_PER_WINDOW).toBe(
+      260,
+    );
+    expect(
+      windowsInThreeHours * CertificateOrderBudget.ORDERS_PER_WINDOW,
+    ).toBeLessThan(CertificateOrderBudget.LETS_ENCRYPT_ORDERS_PER_THREE_HOURS);
+    expect(CertificateOrderBudget.LETS_ENCRYPT_ORDERS_PER_THREE_HOURS).toBe(
+      300,
+    );
+  });
+
+  test("renewals and the primary host may use the whole window, new certificates part of it", () => {
+    expect(
+      CertificateOrderBudget.getLimit(CertificateOrderReason.Renewal),
+    ).toBe(CertificateOrderBudget.ORDERS_PER_WINDOW);
+    expect(
+      CertificateOrderBudget.getLimit(CertificateOrderReason.PrimaryHost),
+    ).toBe(CertificateOrderBudget.ORDERS_PER_WINDOW);
+    expect(
+      CertificateOrderBudget.getLimit(CertificateOrderReason.FirstCertificate),
+    ).toBe(CertificateOrderBudget.NEW_CERTIFICATE_ORDERS_PER_WINDOW);
+    expect(
+      CertificateOrderBudget.getLimit(CertificateOrderReason.Reissue),
+    ).toBe(CertificateOrderBudget.NEW_CERTIFICATE_ORDERS_PER_WINDOW);
+  });
+
+  test("renewals always keep a share no flood of new certificates can take", () => {
+    const keptForRenewals: number =
+      CertificateOrderBudget.ORDERS_PER_WINDOW -
+      CertificateOrderBudget.NEW_CERTIFICATE_ORDERS_PER_WINDOW;
+
+    expect(keptForRenewals).toBe(8);
+    // More than 750 renewals a day, whatever new certificates are ordered.
+    expect(keptForRenewals * 4 * 24).toBeGreaterThan(750);
+  });
+
+  test("a window is 15 minutes, the same on every replica", () => {
+    const start: Date = new Date(
+      Math.floor(NOW.getTime() / WINDOW_IN_MS) * WINDOW_IN_MS,
+    );
+
+    expect(CertificateOrderBudget.getWindowIndex(start)).toBe(
+      CertificateOrderBudget.getWindowIndex(
+        new Date(start.getTime() + WINDOW_IN_MS - 1),
+      ),
+    );
+    expect(
+      CertificateOrderBudget.getWindowIndex(
+        new Date(start.getTime() + WINDOW_IN_MS),
+      ),
+    ).toBe(CertificateOrderBudget.getWindowIndex(start) + 1);
+  });
+});
+
+describe("CertificateOrderBudget.takeSlot", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("new certificates get NEW_CERTIFICATE_ORDERS_PER_WINDOW in a window, and no more", async () => {
+    useInMemoryRedis();
+
+    const answers: Array<boolean> = [];
+
+    for (
+      let i: number = 0;
+      i < CertificateOrderBudget.NEW_CERTIFICATE_ORDERS_PER_WINDOW + 3;
+      i++
+    ) {
+      answers.push(
+        await take(
+          i % 2 === 0
+            ? CertificateOrderReason.FirstCertificate
+            : CertificateOrderReason.Reissue,
+        ),
+      );
+    }
+
+    expect(
+      answers.filter((answer: boolean) => {
+        return answer;
+      }),
+    ).toHaveLength(CertificateOrderBudget.NEW_CERTIFICATE_ORDERS_PER_WINDOW);
+    expect(answers.slice(-3)).toEqual([false, false, false]);
+  });
+
+  /*
+   * One counter holds both limits, and a refusal adds nothing to it: new
+   * certificates refused at their share do not use up what the renewals
+   * may still take.
+   */
+  test("refused new certificates leave the renewals their share", async () => {
+    useInMemoryRedis();
+
+    for (let i: number = 0; i < 50; i++) {
+      await take(CertificateOrderReason.FirstCertificate);
+    }
+
+    const renewals: Array<boolean> = [];
+
+    for (let i: number = 0; i < 20; i++) {
+      renewals.push(await take(CertificateOrderReason.Renewal));
+    }
+
+    expect(
+      renewals.filter((answer: boolean) => {
+        return answer;
+      }),
+    ).toHaveLength(
+      CertificateOrderBudget.ORDERS_PER_WINDOW -
+        CertificateOrderBudget.NEW_CERTIFICATE_ORDERS_PER_WINDOW,
+    );
+  });
+
+  test("renewals that came first leave new certificates only what is under their share", async () => {
+    useInMemoryRedis();
+
+    for (let i: number = 0; i < 10; i++) {
+      expect(await take(CertificateOrderReason.Renewal)).toBe(true);
+    }
+
+    expect(await take(CertificateOrderReason.FirstCertificate)).toBe(true);
+    expect(await take(CertificateOrderReason.FirstCertificate)).toBe(true);
+    expect(await take(CertificateOrderReason.FirstCertificate)).toBe(false);
+
+    // The renewals and the primary host go on to the whole window.
+    for (let i: number = 0; i < 8; i++) {
+      expect(await take(CertificateOrderReason.PrimaryHost)).toBe(true);
+    }
+
+    expect(await take(CertificateOrderReason.Renewal)).toBe(false);
+  });
+
+  test("the next window has a whole budget again", async () => {
+    useInMemoryRedis();
+
+    for (let i: number = 0; i < 25; i++) {
+      await take(CertificateOrderReason.Renewal);
+    }
+
+    expect(await take(CertificateOrderReason.Renewal)).toBe(false);
+    expect(
+      await take(
+        CertificateOrderReason.Renewal,
+        new Date(NOW.getTime() + WINDOW_IN_MS),
+      ),
+    ).toBe(true);
+  });
+
+  test("counts in one key per window, which expires after two windows", async () => {
+    const calls: Array<{
+      namespace: string;
+      key: string;
+      options: { limit: number; expiresInSeconds: number };
+    }> = [];
+
+    jest.spyOn(GlobalCache, "incrementIfBelow").mockImplementation((async (
+      namespace: string,
+      key: string,
+      options: { limit: number; expiresInSeconds: number },
+    ) => {
+      calls.push({ namespace, key, options });
+      return 1;
+    }) as never);
+
+    await take(CertificateOrderReason.FirstCertificate);
+    await take(CertificateOrderReason.Renewal);
+
+    expect(calls).toEqual([
+      {
+        namespace: CertificateOrderBudget.NAMESPACE,
+        key: `window-${CertificateOrderBudget.getWindowIndex(NOW)}`,
+        options: {
+          limit: CertificateOrderBudget.NEW_CERTIFICATE_ORDERS_PER_WINDOW,
+          expiresInSeconds: CertificateOrderBudget.WINDOW_IN_MINUTES * 2 * 60,
+        },
+      },
+      {
+        namespace: CertificateOrderBudget.NAMESPACE,
+        key: `window-${CertificateOrderBudget.getWindowIndex(NOW)}`,
+        options: {
+          limit: CertificateOrderBudget.ORDERS_PER_WINDOW,
+          expiresInSeconds: CertificateOrderBudget.WINDOW_IN_MINUTES * 2 * 60,
+        },
+      },
+    ]);
+  });
+
+  test("without Redis nothing is taken, so nothing is ordered", async () => {
+    const redis: InMemoryRedis = useInMemoryRedis();
+    redis.goDown();
+
+    expect(await take(CertificateOrderReason.Renewal)).toBe(false);
+    expect(await take(CertificateOrderReason.FirstCertificate)).toBe(false);
+  });
+});

package/Tests/Server/Utils/Greenlock/CertificateOrderFailures.test.ts

@@ -0,0 +1,293 @@
+/**
+ * CertificateOrderFailures - the last failed certificate order of each name.
+ *
+ * An order that fails leaves nothing in the database, so a domain whose order
+ * kept failing - a CAA record that leaves Let's Encrypt out, a server Let's
+ * Encrypt cannot reach - showed "Issuing a free certificate" for days, and
+ * the sweeps ordered it again every 15 minutes, each time against the account
+ * the whole installation shares. This record is what the Status column shows,
+ * and what makes the sweeps wait longer after each failure in a row.
+ */
+
+import CertificateOrderFailures, {
+  CertificateOrderFailure,
+} from "../../../../Server/Utils/Greenlock/CertificateOrderFailures";
+import GlobalCache from "../../../../Server/Infrastructure/GlobalCache";
+import OneUptimeDate from "../../../../Types/Date";
+import { InMemoryRedis, useInMemoryRedis } from "./InMemoryRedis";
+import { afterEach, describe, expect, jest, test } from "@jest/globals";
+import type { Mock, SpyInstance } from "jest-mock";
+
+const NOW: Date = new Date("2026-10-03T12:00:00.000Z");
+
+function minutesAfter(date: Date, minutes: number): Date {
+  return OneUptimeDate.addRemoveMinutes(date, minutes);
+}
+
+function failure(
+  failures: number,
+  failedAt: Date = NOW,
+): CertificateOrderFailure {
+  return { error: "Unable to order certificate", failedAt, failures };
+}
+
+describe("CertificateOrderFailures retry delay", () => {
+  test("waits 15 minutes after one failure, doubling with each further one, up to 6 hours", () => {
+    expect(
+      [1, 2, 3, 4, 5, 6, 7, 8, 50, 1000].map((failures: number) => {
+        return CertificateOrderFailures.getRetryDelayInMinutes(failures);
+      }),
+    ).toEqual([15, 30, 60, 120, 240, 360, 360, 360, 360, 360]);
+  });
+
+  test("an odd count is read as one failure", () => {
+    expect(CertificateOrderFailures.getRetryDelayInMinutes(0)).toBe(15);
+    expect(CertificateOrderFailures.getRetryDelayInMinutes(-3)).toBe(15);
+    expect(CertificateOrderFailures.getRetryDelayInMinutes(NaN)).toBe(15);
+  });
+
+  /*
+   * After the first failures the sweeps, every 15 minutes, order a name that
+   * keeps failing a few times an hour, then four times a day - not 96.
+   */
+  test("a name that keeps failing is ordered a few times on the first day, then four times a day", () => {
+    let at: Date = NOW;
+    let failures: number = 0;
+    const attempts: Array<Date> = [];
+    const end: Date = minutesAfter(NOW, 3 * 24 * 60);
+
+    // The sweeps run every 15 minutes; each attempt fails.
+    for (let tick: Date = NOW; tick < end; tick = minutesAfter(tick, 15)) {
+      const record: CertificateOrderFailure | undefined =
+        failures > 0 ? failure(failures, at) : undefined;
+
+      if (!CertificateOrderFailures.isWaitingToRetry(record, tick)) {
+        attempts.push(tick);
+        at = tick;
+        failures++;
+      }
+    }
+
+    const firstDay: number = attempts.filter((attempt: Date) => {
+      return attempt < minutesAfter(NOW, 24 * 60);
+    }).length;
+
+    const thirdDay: number = attempts.filter((attempt: Date) => {
+      return attempt >= minutesAfter(NOW, 2 * 24 * 60);
+    }).length;
+
+    expect(firstDay).toBeLessThanOrEqual(10);
+    expect(thirdDay).toBe(4);
+  });
+
+  test("a name is waiting until its delay is up, minus a little grace for how long the order took", () => {
+    const record: CertificateOrderFailure = failure(1);
+
+    expect(CertificateOrderFailures.getRetryAt(record)).toEqual(
+      minutesAfter(NOW, 15),
+    );
+    expect(
+      CertificateOrderFailures.isWaitingToRetry(record, minutesAfter(NOW, 5)),
+    ).toBe(true);
+    // The sweep on the tick the delay ends on orders it.
+    expect(
+      CertificateOrderFailures.isWaitingToRetry(
+        record,
+        minutesAfter(NOW, 15 - CertificateOrderFailures.RETRY_GRACE_IN_MINUTES),
+      ),
+    ).toBe(false);
+    expect(
+      CertificateOrderFailures.isWaitingToRetry(record, minutesAfter(NOW, 15)),
+    ).toBe(false);
+    expect(CertificateOrderFailures.isWaitingToRetry(undefined, NOW)).toBe(
+      false,
+    );
+  });
+});
+
+describe("CertificateOrderFailures records", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("records a failure by its normalized name, and counts failures in a row", async () => {
+    const redis: InMemoryRedis = useInMemoryRedis();
+
+    await CertificateOrderFailures.record({
+      domain: " Status.Acme.com",
+      error: "Unable to order certificate for status.acme.com.",
+      now: NOW,
+    });
+
+    expect(
+      redis.cache.has(`${CertificateOrderFailures.NAMESPACE}-status.acme.com`),
+    ).toBe(true);
+
+    await CertificateOrderFailures.record({
+      domain: "status.acme.com",
+      error: "Still unable.",
+      now: minutesAfter(NOW, 15),
+    });
+
+    const recorded: Map<string, CertificateOrderFailure> =
+      await CertificateOrderFailures.get(["STATUS.acme.com"]);
+
+    expect(recorded.get("status.acme.com")).toEqual({
+      error: "Still unable.",
+      failedAt: minutesAfter(NOW, 15),
+      failures: 2,
+    });
+  });
+
+  test("keeps a record for a week", async () => {
+    const setString: Mock<(...args: Array<unknown>) => Promise<void>> = jest.fn(
+      async (): Promise<void> => {},
+    );
+    jest.spyOn(GlobalCache, "getString").mockResolvedValue(null as never);
+    jest.spyOn(GlobalCache, "setString").mockImplementation(setString as never);
+
+    await CertificateOrderFailures.record({
+      domain: "status.acme.com",
+      error: "Unable",
+      now: NOW,
+    });
+
+    expect(setString.mock.calls[0]![3]).toEqual({
+      expiresInSeconds: 7 * 24 * 60 * 60,
+    });
+  });
+
+  test("an empty message is recorded as a plain one", async () => {
+    useInMemoryRedis();
+
+    await CertificateOrderFailures.record({
+      domain: "status.acme.com",
+      error: "  ",
+      now: NOW,
+    });
+
+    expect(
+      (await CertificateOrderFailures.get(["status.acme.com"])).get(
+        "status.acme.com",
+      )?.error,
+    ).toBe("We could not order an SSL certificate for this domain.");
+  });
+
+  test("clear forgets a name's failures", async () => {
+    useInMemoryRedis();
+
+    await CertificateOrderFailures.record({
+      domain: "status.acme.com",
+      error: "Unable",
+      now: NOW,
+    });
+    await CertificateOrderFailures.clear("Status.Acme.com");
+
+    expect((await CertificateOrderFailures.get(["status.acme.com"])).size).toBe(
+      0,
+    );
+
+    // The next failure starts a new run of failures.
+    await CertificateOrderFailures.record({
+      domain: "status.acme.com",
+      error: "Unable",
+      now: NOW,
+    });
+
+    expect(
+      (await CertificateOrderFailures.get(["status.acme.com"])).get(
+        "status.acme.com",
+      )?.failures,
+    ).toBe(1);
+  });
+
+  test("reads many names in round trips of a few hundred, and skips what it cannot read", async () => {
+    const redis: InMemoryRedis = useInMemoryRedis();
+
+    const names: Array<string> = Array.from(
+      { length: 1100 },
+      (_value: unknown, index: number) => {
+        return `domain${index}.acme.com`;
+      },
+    );
+
+    await CertificateOrderFailures.record({
+      domain: names[1050]!,
+      error: "Unable",
+      now: NOW,
+    });
+
+    redis.cache.set(
+      `${CertificateOrderFailures.NAMESPACE}-${names[3]}`,
+      "not json",
+    );
+    redis.cache.set(
+      `${CertificateOrderFailures.NAMESPACE}-${names[4]}`,
+      JSON.stringify({ error: 42, failedAt: "yesterday" }),
+    );
+
+    const getStrings: SpyInstance<typeof GlobalCache.getStrings> = jest.spyOn(
+      GlobalCache,
+      "getStrings",
+    );
+
+    const recorded: Map<string, CertificateOrderFailure> =
+      await CertificateOrderFailures.get([...names, names[0]!, ""]);
+
+    expect([...recorded.keys()]).toEqual([names[1050]]);
+    expect(getStrings).toHaveBeenCalledTimes(3);
+  });
+
+  test("never throws: without Redis nothing is recorded, cleared or read", async () => {
+    const redis: InMemoryRedis = useInMemoryRedis();
+    redis.goDown();
+
+    await expect(
+      CertificateOrderFailures.record({
+        domain: "status.acme.com",
+        error: "Unable",
+        now: NOW,
+      }),
+    ).resolves.toBeUndefined();
+    await expect(
+      CertificateOrderFailures.clear("status.acme.com"),
+    ).resolves.toBeUndefined();
+    expect((await CertificateOrderFailures.get(["status.acme.com"])).size).toBe(
+      0,
+    );
+  });
+
+  test("withoutThoseWaitingToRetry keeps what may be ordered now", async () => {
+    useInMemoryRedis();
+
+    await CertificateOrderFailures.record({
+      domain: "failing.acme.com",
+      error: "Unable",
+      now: NOW,
+    });
+    await CertificateOrderFailures.record({
+      domain: "failed-long-ago.acme.com",
+      error: "Unable",
+      now: minutesAfter(NOW, -60),
+    });
+
+    const items: Array<{ fullDomain: string }> = [
+      { fullDomain: "new.acme.com" },
+      { fullDomain: "Failing.Acme.com" },
+      { fullDomain: "failed-long-ago.acme.com" },
+    ];
+
+    expect(
+      await CertificateOrderFailures.withoutThoseWaitingToRetry({
+        items: items,
+        getDomain: (item: { fullDomain: string }): string => {
+          return item.fullDomain;
+        },
+        now: minutesAfter(NOW, 5),
+      }),
+    ).toEqual([
+      { fullDomain: "new.acme.com" },
+      { fullDomain: "failed-long-ago.acme.com" },
+    ]);
+  });
+});

package/Tests/Server/Utils/Greenlock/CertificateOrderLock.test.ts

@@ -0,0 +1,152 @@
+/**
+ * CertificateOrderLock - one certificate order per name at a time, across
+ * every replica, for every kind of order.
+ *
+ * The handle tryLock gives out is what GreenlockUtil.orderCert accepts as
+ * proof that its caller holds the name's lock; anything else is refused. So
+ * it must be the name it was taken for, normalized, and stop counting once
+ * released.
+ */
+
+import CertificateOrderLock, {
+  CertificateOrderLockHandle,
+} from "../../../../Server/Utils/Greenlock/CertificateOrderLock";
+import CertificateOrder from "../../../../Server/Utils/Greenlock/CertificateOrder";
+import Semaphore, {
+  SemaphoreMutex,
+} from "../../../../Server/Infrastructure/Semaphore";
+import { InMemoryRedis, useInMemoryRedis } from "./InMemoryRedis";
+import { afterEach, describe, expect, jest, test } from "@jest/globals";
+
+describe("CertificateOrderLock", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("locks the normalized name, without waiting, for long enough to outlive a slow order", async () => {
+    const redis: InMemoryRedis = useInMemoryRedis();
+
+    const lock: CertificateOrderLockHandle | null =
+      await CertificateOrderLock.tryLock("  Status.Acme.COM ");
+
+    expect(lock?.domain).toBe("status.acme.com");
+    expect(redis.lockRequests).toEqual([
+      expect.objectContaining({
+        key: "status.acme.com",
+        namespace: "CustomDomainCertificateOrder",
+        acquireAttemptsLimit: 1,
+        lockTimeout: 2 * 60 * 1000,
+      }),
+    ]);
+  });
+
+  /*
+   * The namespace is the one CertificateOrder used before every order took
+   * the lock, so an order placed by a replica still running the old code
+   * during a deploy keeps out a new one for the same name.
+   */
+  test("keeps the namespace and timeout CertificateOrder has always used", () => {
+    expect(CertificateOrder.LOCK_NAMESPACE).toBe(
+      CertificateOrderLock.NAMESPACE,
+    );
+    expect(CertificateOrder.LOCK_TIMEOUT_IN_MS).toBe(
+      CertificateOrderLock.TIMEOUT_IN_MS,
+    );
+    expect(CertificateOrderLock.NAMESPACE).toBe("CustomDomainCertificateOrder");
+  });
+
+  test("a name whose lock is held cannot be locked again until it is released", async () => {
+    const redis: InMemoryRedis = useInMemoryRedis();
+
+    const first: CertificateOrderLockHandle | null =
+      await CertificateOrderLock.tryLock("status.acme.com");
+
+    expect(first).not.toBeNull();
+    expect(await CertificateOrderLock.tryLock("STATUS.acme.com")).toBeNull();
+    // Another name is not held up.
+    expect(await CertificateOrderLock.tryLock("other.acme.com")).not.toBeNull();
+
+    await CertificateOrderLock.release(first!);
+
+    expect(
+      redis.heldLocks.has("CustomDomainCertificateOrder-status.acme.com"),
+    ).toBe(false);
+    expect(
+      await CertificateOrderLock.tryLock("status.acme.com"),
+    ).not.toBeNull();
+  });
+
+  test("a handle holds its own name only, and only until it is released", async () => {
+    useInMemoryRedis();
+
+    const lock: CertificateOrderLockHandle | null =
+      await CertificateOrderLock.tryLock("status.acme.com");
+
+    expect(CertificateOrderLock.isHeldFor(lock!, "Status.Acme.com")).toBe(true);
+    expect(CertificateOrderLock.isHeldFor(lock!, "other.acme.com")).toBe(false);
+    expect(
+      CertificateOrderLock.isHeldFor(
+        { domain: "status.acme.com" } as CertificateOrderLockHandle,
+        "status.acme.com",
+      ),
+    ).toBe(false);
+    expect(CertificateOrderLock.isHeldFor(undefined, "status.acme.com")).toBe(
+      false,
+    );
+
+    await CertificateOrderLock.release(lock!);
+
+    expect(CertificateOrderLock.isHeldFor(lock!, "status.acme.com")).toBe(
+      false,
+    );
+  });
+
+  test("a lock Redis loses while it is held is held no more, and is known as lost", async () => {
+    const redis: InMemoryRedis = useInMemoryRedis();
+
+    const lock: CertificateOrderLockHandle | null =
+      await CertificateOrderLock.tryLock("status.acme.com");
+
+    expect(CertificateOrderLock.isHeldFor(lock!, "status.acme.com")).toBe(true);
+    expect(CertificateOrderLock.wasLost(lock!, "status.acme.com")).toBe(false);
+
+    (
+      redis.lockRequests[0] as unknown as {
+        onLockLost: (err: Error) => void;
+      }
+    ).onLockLost(new Error("refresh failed"));
+
+    expect(CertificateOrderLock.isHeldFor(lock!, "status.acme.com")).toBe(
+      false,
+    );
+    expect(CertificateOrderLock.wasLost(lock!, "status.acme.com")).toBe(true);
+    // Lost for its own name only.
+    expect(CertificateOrderLock.wasLost(lock!, "other.acme.com")).toBe(false);
+
+    // Releasing it afterwards never throws.
+    await expect(CertificateOrderLock.release(lock!)).resolves.toBeUndefined();
+  });
+
+  test("without Redis no lock is taken, so nothing is ordered", async () => {
+    const redis: InMemoryRedis = useInMemoryRedis();
+    redis.goDown();
+
+    expect(await CertificateOrderLock.tryLock("status.acme.com")).toBeNull();
+  });
+
+  test("a release that fails, or a second release, never throws", async () => {
+    useInMemoryRedis();
+
+    const lock: CertificateOrderLockHandle | null =
+      await CertificateOrderLock.tryLock("status.acme.com");
+
+    jest.spyOn(Semaphore, "release").mockImplementation((async (
+      _mutex: SemaphoreMutex,
+    ) => {
+      throw new Error("Valkey went away");
+    }) as never);
+
+    await expect(CertificateOrderLock.release(lock!)).resolves.toBeUndefined();
+    await expect(CertificateOrderLock.release(lock!)).resolves.toBeUndefined();
+  });
+});