@oneuptime/common 14.0.13 → 14.0.14

package/Tests/Server/Utils/Greenlock/CertificateRenewalSafety.test.ts

@@ -0,0 +1,368 @@
+/**
+ * A renewal run never costs a domain its working certificate, and never
+ * orders more than it must.
+ *
+ *   - A CNAME check that fails while the certificate is still valid keeps the
+ *     certificate; the renewal is tried again on a later run. It used to
+ *     remove it, so a moment's DNS trouble during a renewal took the domain
+ *     off HTTPS with weeks left to run. Only an expired certificate - it
+ *     serves nobody - is removed when the check fails.
+ *   - The CNAME is checked once per renewal. The run checked it, then the
+ *     order checked it again.
+ *   - Each renewal runs under the name's order lock, and looks the
+ *     certificate up again under it: a reissue or re-order of the same name
+ *     may be running, or have just renewed it.
+ *   - A name whose renewals keep failing waits longer after each failure in
+ *     a row instead of taking a slot in every run.
+ *
+ * The table is a spy, Redis is in memory, and orderCert only records what it
+ * is asked.
+ */
+
+import GreenlockUtil from "../../../../Server/Utils/Greenlock/Greenlock";
+import CertificateOrderLock from "../../../../Server/Utils/Greenlock/CertificateOrderLock";
+import CertificateOrderFailures, {
+  CertificateOrderFailure,
+} from "../../../../Server/Utils/Greenlock/CertificateOrderFailures";
+import { CertificateOrderReason } from "../../../../Server/Utils/Greenlock/CertificateOrderBudget";
+import { CertificateOrderOutcome } from "../../../../Server/Utils/Greenlock/CertificateOrderOutcome";
+import AcmeCertificateService from "../../../../Server/Services/AcmeCertificateService";
+import QueryHelper from "../../../../Server/Types/Database/QueryHelper";
+import AcmeCertificate from "../../../../Models/DatabaseModels/AcmeCertificate";
+import OneUptimeDate from "../../../../Types/Date";
+import { InMemoryRedis, useInMemoryRedis } from "./InMemoryRedis";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  jest,
+  test,
+} from "@jest/globals";
+
+type OrderCall = {
+  domain: string;
+  reason: CertificateOrderReason;
+  validateCname: unknown;
+  heldLock: boolean;
+};
+
+type Run = {
+  // The certificate table: name -> expiry.
+  table: Map<string, Date>;
+  orders: Array<OrderCall>;
+  removed: Array<string>;
+  notified: Array<string>;
+  cnameChecks: Array<string>;
+  brokenCnames: Set<string>;
+  redis: InMemoryRedis;
+};
+
+const NOW: Date = new Date("2026-10-03T12:00:00.000Z");
+
+function inDays(days: number): Date {
+  return OneUptimeDate.addRemoveDays(NOW, days);
+}
+
+function setUpRun(certificates: Record<string, number>): Run {
+  const run: Run = {
+    table: new Map<string, Date>(
+      Object.entries(certificates).map(([domain, days]: [string, number]) => {
+        return [domain, inDays(days)];
+      }),
+    ),
+    orders: [],
+    removed: [],
+    notified: [],
+    cnameChecks: [],
+    brokenCnames: new Set<string>(),
+    redis: useInMemoryRedis(),
+  };
+
+  jest.spyOn(OneUptimeDate, "getCurrentDate").mockReturnValue(NOW);
+
+  jest.spyOn(QueryHelper, "any").mockImplementation(((
+    values: Array<string>,
+  ) => {
+    return { inList: values.map(String) };
+  }) as never);
+
+  // The due rows by expiry, or a lookup by name.
+  jest
+    .spyOn(AcmeCertificateService, "findBy")
+    .mockImplementation((async (call: {
+      query: { domain?: { inList: Array<string> } };
+      skip?: number;
+    }) => {
+      if (call.skip) {
+        return [];
+      }
+
+      return [...run.table.entries()]
+        .filter(([domain]: [string, Date]) => {
+          return (
+            !call.query.domain || call.query.domain.inList.includes(domain)
+          );
+        })
+        .sort((a: [string, Date], b: [string, Date]) => {
+          return a[1].getTime() - b[1].getTime();
+        })
+        .map(([domain, expiresAt]: [string, Date]) => {
+          return { domain, expiresAt } as unknown as AcmeCertificate;
+        });
+    }) as never);
+
+  jest
+    .spyOn(AcmeCertificateService, "deleteBy")
+    .mockImplementation((async (call: { query: { domain: string } }) => {
+      run.removed.push(call.query.domain);
+      run.table.delete(call.query.domain);
+      return 1;
+    }) as never);
+
+  jest
+    .spyOn(GreenlockUtil, "orderCert")
+    .mockImplementation(
+      async (data: {
+        domain: string;
+        reason: CertificateOrderReason;
+        validateCname: unknown;
+        lock?: unknown;
+      }): Promise<CertificateOrderOutcome> => {
+        run.orders.push({
+          domain: data.domain,
+          reason: data.reason,
+          validateCname: data.validateCname,
+          heldLock: CertificateOrderLock.isHeldFor(
+            data.lock as never,
+            data.domain,
+          ),
+        });
+        run.table.set(data.domain, inDays(90));
+        return CertificateOrderOutcome.Ordered;
+      },
+    );
+
+  return run;
+}
+
+async function renew(run: Run): Promise<void> {
+  await GreenlockUtil.renewAllCertsWhichAreExpiringSoon({
+    getOwnedDomains: async (domains: Array<string>) => {
+      return domains;
+    },
+    validateCname: async (domain: string): Promise<boolean> => {
+      run.cnameChecks.push(domain);
+      return !run.brokenCnames.has(domain);
+    },
+    notifyDomainRemoved: async (domain: string): Promise<void> => {
+      run.notified.push(domain);
+    },
+  });
+}
+
+function orderedDomains(run: Run): Array<string> {
+  return run.orders
+    .map((order: OrderCall) => {
+      return order.domain;
+    })
+    .sort();
+}
+
+describe("a renewal run", () => {
+  beforeEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("renews under the name's lock, as a renewal, with the CNAME checked once", async () => {
+    const run: Run = setUpRun({ "status.acme.com": 3 });
+
+    await renew(run);
+
+    expect(run.orders).toEqual([
+      {
+        domain: "status.acme.com",
+        reason: CertificateOrderReason.Renewal,
+        validateCname: null,
+        heldLock: true,
+      },
+    ]);
+    expect(run.cnameChecks).toEqual(["status.acme.com"]);
+    expect(run.redis.heldLocks.size).toBe(0);
+  });
+
+  test("a CNAME check that fails while the certificate is valid keeps it, records why, and a later run renews it", async () => {
+    const run: Run = setUpRun({ "status.acme.com": 3, "other.acme.com": 4 });
+    run.brokenCnames.add("status.acme.com");
+
+    await renew(run);
+
+    expect(run.removed).toEqual([]);
+    expect(run.notified).toEqual([]);
+    expect(orderedDomains(run)).toEqual(["other.acme.com"]);
+    expect(run.table.get("status.acme.com")).toEqual(inDays(3));
+
+    // The Status column says why; the name waits before the next attempt.
+    const failure: CertificateOrderFailure | undefined = (
+      await CertificateOrderFailures.get(["status.acme.com"])
+    ).get("status.acme.com");
+
+    expect(failure?.error).toContain("CNAME record could not be verified");
+    expect(failure?.failures).toBe(1);
+
+    run.brokenCnames.clear();
+
+    // Its first retry delay is up.
+    jest
+      .spyOn(OneUptimeDate, "getCurrentDate")
+      .mockReturnValue(OneUptimeDate.addRemoveMinutes(NOW, 15));
+
+    await renew(run);
+
+    expect(orderedDomains(run)).toEqual(["other.acme.com", "status.acme.com"]);
+  });
+
+  /*
+   * Review: a valid certificate whose record really is gone was skipped
+   * without a trace, so - due, and first by expiry - it took a slot of every
+   * run until it expired, while the renewals behind it waited.
+   */
+  test("certificates whose CNAME keeps failing do not take the slots of the renewals behind them", async () => {
+    const domains: Record<string, number> = {};
+
+    // A full run of moved-away domains, closest to expiry...
+    for (let i: number = 0; i < GreenlockUtil.RENEW_MAX_PER_RUN; i++) {
+      domains[`moved${String(i).padStart(2, "0")}.acme.com`] = 1;
+    }
+
+    // ...and the renewals behind them.
+    for (let i: number = 0; i < 3; i++) {
+      domains[`renew${i}.acme.com`] = 10;
+    }
+
+    const run: Run = setUpRun(domains);
+
+    for (let i: number = 0; i < GreenlockUtil.RENEW_MAX_PER_RUN; i++) {
+      run.brokenCnames.add(`moved${String(i).padStart(2, "0")}.acme.com`);
+    }
+
+    // The first runs try the moved-away ones, and fill up on them...
+    await renew(run);
+
+    jest
+      .spyOn(OneUptimeDate, "getCurrentDate")
+      .mockReturnValue(OneUptimeDate.addRemoveMinutes(NOW, 15));
+
+    await renew(run);
+
+    expect(run.orders).toEqual([]);
+
+    /*
+     * ...after a second failure in a row they wait 30 minutes, so the run in
+     * between is the renewals' - and with each failure they wait longer.
+     */
+    jest
+      .spyOn(OneUptimeDate, "getCurrentDate")
+      .mockReturnValue(OneUptimeDate.addRemoveMinutes(NOW, 30));
+
+    await renew(run);
+
+    expect(orderedDomains(run)).toEqual([
+      "renew0.acme.com",
+      "renew1.acme.com",
+      "renew2.acme.com",
+    ]);
+    expect(run.removed).toEqual([]);
+  });
+
+  test("a CNAME check that fails on an expired certificate removes it, and tells its owner", async () => {
+    const run: Run = setUpRun({ "status.acme.com": -2 });
+    run.brokenCnames.add("status.acme.com");
+
+    await renew(run);
+
+    expect(run.removed).toEqual(["status.acme.com"]);
+    expect(run.notified).toEqual(["status.acme.com"]);
+    expect(run.orders).toEqual([]);
+  });
+
+  test("a name being ordered right now - a reissue, a Check now - is left to a later run", async () => {
+    const run: Run = setUpRun({ "status.acme.com": 3, "other.acme.com": 4 });
+    run.redis.heldLocks.add(
+      `${CertificateOrderLock.NAMESPACE}-status.acme.com`,
+    );
+
+    await renew(run);
+
+    expect(orderedDomains(run)).toEqual(["other.acme.com"]);
+    expect(run.cnameChecks).toEqual(["other.acme.com"]);
+    // The other order's lock is left alone.
+    expect(
+      run.redis.heldLocks.has(
+        `${CertificateOrderLock.NAMESPACE}-status.acme.com`,
+      ),
+    ).toBe(true);
+  });
+
+  test("a certificate another order renewed since the run read the table is not renewed again", async () => {
+    const run: Run = setUpRun({ "status.acme.com": 3 });
+
+    const findBy: (call: unknown) => Promise<unknown> = (
+      AcmeCertificateService.findBy as unknown as {
+        getMockImplementation: () => (call: unknown) => Promise<unknown>;
+      }
+    ).getMockImplementation();
+
+    // The run reads the due rows; a reissue renews the name; then the lookup under the lock.
+    jest
+      .spyOn(AcmeCertificateService, "findBy")
+      .mockImplementation((async (call: { query: { domain?: unknown } }) => {
+        if (call.query.domain) {
+          run.table.set("status.acme.com", inDays(90));
+        }
+
+        return await findBy(call);
+      }) as never);
+
+    await renew(run);
+
+    expect(run.orders).toEqual([]);
+    expect(run.cnameChecks).toEqual([]);
+  });
+
+  test("a name whose renewal just failed waits, and does not take a slot from the others", async () => {
+    const domains: Record<string, number> = {};
+
+    for (let i: number = 0; i <= GreenlockUtil.RENEW_MAX_PER_RUN; i++) {
+      domains[`status${String(i).padStart(2, "0")}.acme.com`] = 1 + i;
+    }
+
+    const run: Run = setUpRun(domains);
+
+    // The one closest to expiry failed a moment ago.
+    await CertificateOrderFailures.record({
+      domain: "status00.acme.com",
+      error: "Unable to order certificate for status00.acme.com.",
+      now: NOW,
+    });
+
+    await renew(run);
+
+    expect(run.orders).toHaveLength(GreenlockUtil.RENEW_MAX_PER_RUN);
+    expect(orderedDomains(run)).not.toContain("status00.acme.com");
+
+    // Once its delay is up, it is renewed first again.
+    jest
+      .spyOn(OneUptimeDate, "getCurrentDate")
+      .mockReturnValue(OneUptimeDate.addRemoveMinutes(NOW, 15));
+    run.orders.length = 0;
+
+    await renew(run);
+
+    expect(orderedDomains(run)).toContain("status00.acme.com");
+  });
+});

package/Tests/Server/Utils/Greenlock/CertificateRenewalSchedule.test.ts

@@ -20,14 +20,25 @@
  * the reactive sweep spends from the same allowance.
  *
  * AcmeCertificateService.findBy is spied on, so no database is touched, and
- * orderCert is spied on so no ACME order is ever attempted.
+ * orderCert is spied on so no ACME order is ever attempted. Every run here
+ * owns every certificate it is shown; which certificates a run may touch at
+ * all is pinned in CertificateRenewalOwnership.test.ts.
  */
 
 import GreenlockUtil from "../../../../Server/Utils/Greenlock/Greenlock";
+import { CertificateOrderOutcome } from "../../../../Server/Utils/Greenlock/CertificateOrderOutcome";
 import AcmeCertificateService from "../../../../Server/Services/AcmeCertificateService";
 import AcmeCertificate from "../../../../Models/DatabaseModels/AcmeCertificate";
 import OneUptimeDate from "../../../../Types/Date";
-import { afterEach, describe, expect, test, jest } from "@jest/globals";
+import { useInMemoryRedis } from "./InMemoryRedis";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  test,
+  jest,
+} from "@jest/globals";
 
 function certificateExpiringInDays(
   domain: string,
@@ -42,6 +53,11 @@ function certificateExpiringInDays(
   } as unknown as AcmeCertificate;
 }
 
+// The caller owns every domain it is asked about.
+async function ownsEveryDomain(domains: Array<string>): Promise<Array<string>> {
+  return domains;
+}
+
 /*
  * Drive a renewal run over a fixed set of certificates and report which domains
  * it actually tried to order. The CNAME check is stubbed valid so the run
@@ -58,11 +74,15 @@ async function domainsRenewedFor(
 
   jest
     .spyOn(GreenlockUtil, "orderCert")
-    .mockImplementation(async (data: { domain: string }): Promise<void> => {
-      ordered.push(data.domain);
-    });
+    .mockImplementation(
+      async (data: { domain: string }): Promise<CertificateOrderOutcome> => {
+        ordered.push(data.domain);
+        return CertificateOrderOutcome.Ordered;
+      },
+    );
 
   await GreenlockUtil.renewAllCertsWhichAreExpiringSoon({
+    getOwnedDomains: ownsEveryDomain,
     validateCname: async (): Promise<boolean> => {
       return true;
     },
@@ -131,6 +151,11 @@ describe("Certificate renewal lead time", () => {
 });
 
 describe("Certificate renewal run", () => {
+  beforeEach(() => {
+    // Each renewal takes its name's order lock.
+    useInMemoryRedis();
+  });
+
   afterEach(() => {
     jest.restoreAllMocks();
   });
@@ -229,14 +254,18 @@ describe("Certificate renewal run", () => {
 
     jest
       .spyOn(GreenlockUtil, "orderCert")
-      .mockImplementation(async (data: { domain: string }): Promise<void> => {
-        if (data.domain === "broken.example.com") {
-          throw new Error("CA refused the order");
-        }
-        ordered.push(data.domain);
-      });
+      .mockImplementation(
+        async (data: { domain: string }): Promise<CertificateOrderOutcome> => {
+          if (data.domain === "broken.example.com") {
+            throw new Error("CA refused the order");
+          }
+          ordered.push(data.domain);
+          return CertificateOrderOutcome.Ordered;
+        },
+      );
 
     await GreenlockUtil.renewAllCertsWhichAreExpiringSoon({
+      getOwnedDomains: ownsEveryDomain,
       validateCname: async (): Promise<boolean> => {
         return true;
       },
@@ -248,3 +277,151 @@ describe("Certificate renewal run", () => {
     expect(ordered).toContain("healthy.example.com");
   });
 });
+
+describe("GreenlockUtil.isDueForRenewal", () => {
+  const now: Date = OneUptimeDate.getCurrentDate();
+
+  test("a certificate inside its own lead time is due, an expired one too", () => {
+    expect(
+      GreenlockUtil.isDueForRenewal(
+        certificateExpiringInDays("status.aleyant.com", 1),
+        now,
+      ),
+    ).toBe(true);
+    expect(
+      GreenlockUtil.isDueForRenewal(
+        certificateExpiringInDays("status.aleyant.com", -3),
+        now,
+      ),
+    ).toBe(true);
+  });
+
+  test("a freshly issued certificate is not due", () => {
+    expect(
+      GreenlockUtil.isDueForRenewal(
+        certificateExpiringInDays("status.aleyant.com", 89),
+        now,
+      ),
+    ).toBe(false);
+  });
+
+  test("the boundary is the domain's own lead time", () => {
+    const domain: string = "status.aleyant.com";
+    const leadTime: number = GreenlockUtil.getRenewalLeadTimeInDays(domain);
+
+    expect(
+      GreenlockUtil.isDueForRenewal(
+        certificateExpiringInDays(domain, leadTime - 1),
+        now,
+      ),
+    ).toBe(true);
+    expect(
+      GreenlockUtil.isDueForRenewal(
+        certificateExpiringInDays(domain, leadTime + 1),
+        now,
+      ),
+    ).toBe(false);
+  });
+
+  test("a row without a domain or an expiry is never due", () => {
+    expect(GreenlockUtil.isDueForRenewal({ expiresAt: now }, now)).toBe(false);
+    expect(
+      GreenlockUtil.isDueForRenewal({ domain: "status.aleyant.com" }, now),
+    ).toBe(false);
+  });
+});
+
+/*
+ * The capped sweeps pick afresh every run: a domain whose order keeps
+ * failing must not hold a slot every run while the domains behind it never
+ * get one - however the list grows or shrinks between runs.
+ */
+describe("GreenlockUtil.pickForThisRun", () => {
+  const FIFTEEN_MINUTES_IN_MS: number = 15 * 60 * 1000;
+  const firstRun: Date = new Date(1_800_000_000_000);
+
+  function nthRun(n: number): Date {
+    return new Date(firstRun.getTime() + n * FIFTEEN_MINUTES_IN_MS);
+  }
+
+  function pick(items: Array<string>, max: number, now: Date): Array<string> {
+    return GreenlockUtil.pickForThisRun({
+      items: items,
+      max: max,
+      now: now,
+      getKey: (item: string): string => {
+        return item;
+      },
+    });
+  }
+
+  const items: Array<string> = Array.from(
+    { length: 12 },
+    (_value: unknown, index: number) => {
+      return "d" + String(index).padStart(2, "0") + ".example.com";
+    },
+  );
+
+  test("takes everything when everything fits", () => {
+    expect(pick(items.slice(0, 3), 5, firstRun)).toEqual(items.slice(0, 3));
+    expect(pick([], 5, firstRun)).toEqual([]);
+  });
+
+  test("takes at most max distinct items", () => {
+    const taken: Array<string> = pick(items, 5, firstRun);
+
+    expect(taken).toHaveLength(5);
+    expect(new Set(taken).size).toBe(5);
+  });
+
+  test("a different pick in the next run", () => {
+    expect(pick(items, 5, nthRun(0))).not.toEqual(pick(items, 5, nthRun(1)));
+  });
+
+  test("the same pick for the whole run, on every replica, whatever order the items arrive in", () => {
+    const reversed: Array<string> = [...items].reverse();
+
+    expect(pick(reversed, 5, nthRun(3))).toEqual(pick(items, 5, nthRun(3)));
+    expect(pick(items, 5, new Date(nthRun(3).getTime() + 60 * 1000))).toEqual(
+      pick(items, 5, nthRun(3)),
+    );
+  });
+
+  test("every item is picked within a few runs, even when the items picked first keep failing", () => {
+    const picked: Set<string> = new Set<string>();
+
+    for (let run: number = 0; run < 12; run++) {
+      for (const item of pick(items, 5, nthRun(run))) {
+        picked.add(item);
+      }
+    }
+
+    expect([...picked].sort()).toEqual(items);
+  });
+
+  /*
+   * The case that broke a fixed rotation: items that succeed leave the list,
+   * so the ones behind them move up. Every item that is still waiting keeps
+   * its chance in every run.
+   */
+  test("every item is picked within a few runs while the list shrinks", () => {
+    let waiting: Array<string> = [...items];
+
+    for (let run: number = 0; run < 12 && waiting.length > 0; run++) {
+      const taken: Array<string> = pick(waiting, 5, nthRun(run));
+
+      // Two of every pick succeed and leave the list; the rest keep failing.
+      const succeeded: Set<string> = new Set<string>(taken.slice(0, 2));
+
+      waiting = waiting.filter((item: string) => {
+        return !succeeded.has(item);
+      });
+    }
+
+    expect(waiting).toEqual([]);
+  });
+
+  test("a max of zero takes nothing", () => {
+    expect(pick(items, 0, firstRun)).toEqual([]);
+  });
+});