@oneuptime/common 14.0.13 → 14.0.14

package/Server/Services/DashboardDomainService.ts

@@ -3,6 +3,14 @@ import DeleteBy from "../Types/Database/DeleteBy";
 import UpdateBy from "../Types/Database/UpdateBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import GreenlockUtil from "../Utils/Greenlock/Greenlock";
+import CertificateOrder, {
+  CertificateOrderOutcome,
+} from "../Utils/Greenlock/CertificateOrder";
+import CustomDomainOrders from "../Utils/Greenlock/CustomDomainOrders";
+import { CertificateOrderLockHandle } from "../Utils/Greenlock/CertificateOrderLock";
+import { CertificateOrderReason } from "../Utils/Greenlock/CertificateOrderBudget";
+import CertificateOrderFailures from "../Utils/Greenlock/CertificateOrderFailures";
+import CertificateReissueOrder from "../Utils/Greenlock/CertificateReissueOrder";
 import logger, { LogAttributes } from "../Utils/Logger";
 import DatabaseService from "./DatabaseService";
 import DomainService from "./DomainService";
@@ -15,19 +23,47 @@ import ObjectID from "../../Types/ObjectID";
 import AcmeCertificate from "../../Models/DatabaseModels/AcmeCertificate";
 import DomainModel from "../../Models/DatabaseModels/Domain";
 import DashboardDomain from "../../Models/DatabaseModels/DashboardDomain";
-import AcmeCertificateService from "./AcmeCertificateService";
 import Telemetry, { Span } from "../Utils/Telemetry";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { DashboardCNameRecord } from "../EnvironmentConfig";
 import Domain from "../Types/Domain";
 import OneUptimeDate from "../../Types/Date";
 import CertificateReissueUtil from "../../Utils/CertificateReissue";
-import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
 import QueryHelper from "../Types/Database/QueryHelper";
+import ArrayUtil from "../../Utils/Array";
+import { CustomDomainVerificationResult } from "../../Types/CustomDomain/CustomDomainVerification";
+import { CustomDomainCertificate } from "../../Types/CustomDomain/CustomDomainCertificates";
 
 const DASHBOARD_DOMAIN_EGRESS_LABEL: string = "Dashboard domain";
 
 export class Service extends DatabaseService<DashboardDomain> {
+  /*
+   * How many certificates the dashboard sweeps may order: the first orders
+   * and the re-orders of a certificate that has gone missing together, in
+   * one 15-minute window (SWEEP_ORDER_BUDGET, CertificateOrder.takeOrderSlot)
+   * - and, separately, how many the dashboard renewal run renews per run.
+   *
+   * Every order spends from the one Let's Encrypt account the whole
+   * installation shares - 300 new orders per account per three hours - and
+   * status page certificates, the larger fleet, are ordered and renewed from
+   * it too. These caps keep a backlog - such as the first runs re-ordering
+   * every dashboard certificate the status page renewal job used to delete -
+   * worked off a few domains per run rather than in one burst, and none of
+   * them can order beyond the installation's own budget for that account
+   * (CertificateOrderBudget), which every order of every kind draws from.
+   */
+  public static readonly ORDER_MAX_PER_RUN: number = 5;
+
+  // The budget the dashboard ordering sweeps share (CertificateOrder.takeOrderSlot).
+  public static readonly SWEEP_ORDER_BUDGET: string = "DashboardDomainSweeps";
+
+  /*
+   * How many domains the verification and provisioning sweeps check at once.
+   * Each check is a bounded request out to a customer domain; the same
+   * figure the status page provisioning sweep uses.
+   */
+  public static readonly DOMAIN_CHECK_CONCURRENCY: number = 10;
+
   public constructor() {
     super(DashboardDomain);
   }
@@ -171,9 +207,81 @@ export class Service extends DatabaseService<DashboardDomain> {
     await GreenlockUtil.removeDomain(domain);
   }
 
+  /*
+   * The domain with its full name and whether it serves an uploaded
+   * certificate, read once when the caller did not have them both.
+   */
+  private async getDomainToOrder(
+    dashboardDomain: DashboardDomain,
+  ): Promise<DashboardDomain> {
+    if (!dashboardDomain.id) {
+      throw new BadDataException(
+        "Domain ID is required to order a certificate",
+      );
+    }
+
+    if (
+      dashboardDomain.fullDomain &&
+      dashboardDomain.isCustomCertificate !== undefined
+    ) {
+      return dashboardDomain;
+    }
+
+    const fetchedDashboardDomain: DashboardDomain | null = await this.findOneBy(
+      {
+        query: {
+          _id: dashboardDomain.id.toString(),
+        },
+        select: {
+          _id: true,
+          fullDomain: true,
+          isCustomCertificate: true,
+          projectId: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      },
+    );
+
+    if (!fetchedDashboardDomain) {
+      throw new BadDataException("DomainModel not found");
+    }
+
+    if (!fetchedDashboardDomain.fullDomain) {
+      throw new BadDataException(
+        "Unable to order certificate because domain is null",
+      );
+    }
+
+    return fetchedDashboardDomain;
+  }
+
+  /*
+   * Order this domain's certificate and record it: a first certificate
+   * (the default), or a reissue. GreenlockUtil.orderCert places it under the
+   * name's order lock - the one handed in, or its own - and within the
+   * installation's Let's Encrypt allowance; the outcome says whether it
+   * ordered. A CNAME check that fails refuses the order and no longer
+   * deletes the domain's certificate.
+   */
   @CaptureSpan()
-  public async orderCert(dashboardDomain: DashboardDomain): Promise<void> {
-    return Telemetry.startActiveSpan<Promise<void>>({
+  public async orderCert(
+    dashboardDomain: DashboardDomain,
+    options?: {
+      reason?: CertificateOrderReason | undefined;
+      // The name's order lock, when the caller holds it.
+      lock?: CertificateOrderLockHandle | undefined;
+      // The caller has just found the domain's CNAME record.
+      cnameVerifiedJustNow?: boolean | undefined;
+      /*
+       * The caller's share of orders (CertificateOrder.getOrderShare),
+       * asked once the CNAME check has passed.
+       */
+      mayOrder?: (() => Promise<boolean>) | undefined;
+    },
+  ): Promise<CertificateOrderOutcome> {
+    return Telemetry.startActiveSpan<Promise<CertificateOrderOutcome>>({
       name: "DashboardDomainService.orderCert",
       options: {
         attributes: {
@@ -181,64 +289,54 @@ export class Service extends DatabaseService<DashboardDomain> {
           _id: dashboardDomain.id?.toString(),
         },
       },
-      fn: async (span: Span): Promise<void> => {
+      fn: async (span: Span): Promise<CertificateOrderOutcome> => {
         try {
-          if (!dashboardDomain.fullDomain) {
-            const fetchedDashboardDomain: DashboardDomain | null =
-              await this.findOneBy({
-                query: {
-                  _id: dashboardDomain.id!.toString(),
-                },
-                select: {
-                  _id: true,
-                  fullDomain: true,
-                },
-                props: {
-                  isRoot: true,
-                },
-              });
-
-            if (!fetchedDashboardDomain) {
-              throw new BadDataException("DomainModel not found");
-            }
-
-            dashboardDomain = fetchedDashboardDomain;
-          }
+          const domainToOrder: DashboardDomain =
+            await this.getDomainToOrder(dashboardDomain);
 
-          if (!dashboardDomain.fullDomain) {
-            throw new BadDataException(
-              "Unable to order certificate because domain is null",
-            );
-          }
-
-          logger.debug(
-            "Ordering SSL for domain: " + dashboardDomain.fullDomain,
-            { fullDomain: dashboardDomain.fullDomain } as LogAttributes,
-          );
+          logger.debug("Ordering SSL for domain: " + domainToOrder.fullDomain, {
+            fullDomain: domainToOrder.fullDomain,
+          } as LogAttributes);
 
-          await GreenlockUtil.orderCert({
-            domain: dashboardDomain.fullDomain as string,
-            validateCname: async (fullDomain: string) => {
-              return await this.isCnameValid(fullDomain);
-            },
-          });
+          const outcome: CertificateOrderOutcome =
+            await GreenlockUtil.orderCert({
+              domain: domainToOrder.fullDomain as string,
+              reason:
+                options?.reason || CertificateOrderReason.FirstCertificate,
+              validateCname: options?.cnameVerifiedJustNow
+                ? null
+                : async (fullDomain: string) => {
+                    return await this.isCnameValid(fullDomain);
+                  },
+              lock: options?.lock,
+              mayOrder: options?.mayOrder,
+            });
 
-          logger.debug(
-            "SSL ordered for domain: " + dashboardDomain.fullDomain,
-            { fullDomain: dashboardDomain.fullDomain } as LogAttributes,
-          );
+          if (outcome === CertificateOrderOutcome.Ordered) {
+            logger.debug(
+              "SSL ordered for domain: " + domainToOrder.fullDomain,
+              { fullDomain: domainToOrder.fullDomain } as LogAttributes,
+            );
 
-          await this.updateOneById({
-            id: dashboardDomain.id!,
-            data: {
-              isSslOrdered: true,
-            },
-            props: {
-              isRoot: true,
-            },
-          });
+            /*
+             * Verified as well as ordered: Let's Encrypt has just fetched
+             * its challenge from this domain, so the domain reaches us.
+             */
+            await this.updateOneById({
+              id: domainToOrder.id!,
+              data: {
+                isCnameVerified: true,
+                isSslOrdered: true,
+              },
+              props: {
+                isRoot: true,
+              },
+            });
+          }
 
           Telemetry.endSpan(span);
+
+          return outcome;
         } catch (err) {
           Telemetry.recordExceptionMarkSpanAsErrorAndEndSpan({
             span,
@@ -251,6 +349,146 @@ export class Service extends DatabaseService<DashboardDomain> {
     });
   }
 
+  /*
+   * Records these domains as ordered: they have a certificate, ordered
+   * before, so nothing is ordered. One write for all of them, and only on
+   * domains that are still verified (see StatusPageDomainService's twin).
+   */
+  private async recordCertificatesAsOrdered(
+    domainIds: Array<ObjectID>,
+  ): Promise<void> {
+    if (domainIds.length === 0) {
+      return;
+    }
+
+    await this.updateBy({
+      query: {
+        _id: QueryHelper.any(
+          domainIds.map((id: ObjectID) => {
+            return id.toString();
+          }),
+        ),
+        isCnameVerified: true,
+        isSslOrdered: false,
+      },
+      data: {
+        isSslOrdered: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+
+  /*
+   * Order this domain's free certificate unless it has one already, or one
+   * is being ordered for it right now (CertificateOrder.orderIfMissing): the
+   * one way a first certificate is ordered, whether by Check now, by the
+   * dashboard sweeps or by the order-ssl API, so a name is never ordered
+   * twice. Never for a domain that serves a certificate its owner uploaded.
+   * Throws what the order throws, so a caller can show why.
+   */
+  @CaptureSpan()
+  public async orderCertIfMissing(
+    dashboardDomain: DashboardDomain,
+    options?: {
+      // An order of the sweeps: it draws from their shared window budget.
+      fromSweep?: boolean | undefined;
+      // An order somebody asked for: it draws from the on-demand budget.
+      onDemand?: boolean | undefined;
+      // The caller has just found the domain's CNAME record.
+      cnameVerifiedJustNow?: boolean | undefined;
+      // Order for an expired certificate too (Check now).
+      renewIfExpired?: boolean | undefined;
+    },
+  ): Promise<CertificateOrderOutcome> {
+    const domainToOrder: DashboardDomain =
+      await this.getDomainToOrder(dashboardDomain);
+
+    if (domainToOrder.isCustomCertificate) {
+      throw new BadDataException(
+        "This domain uses a certificate you uploaded, so there is no free SSL certificate to order for it.",
+      );
+    }
+
+    const domainId: ObjectID = domainToOrder.id!;
+
+    return await CertificateOrder.orderIfMissing({
+      domain: domainToOrder.fullDomain as string,
+      renewIfExpired: options?.renewIfExpired,
+      recordAsOrdered: async (): Promise<void> => {
+        await this.recordCertificatesAsOrdered([domainId]);
+      },
+      order: async (
+        lock: CertificateOrderLockHandle,
+      ): Promise<CertificateOrderOutcome> => {
+        return await this.orderCert(domainToOrder, {
+          reason: CertificateOrderReason.FirstCertificate,
+          lock: lock,
+          cnameVerifiedJustNow: options?.cnameVerifiedJustNow,
+          mayOrder: CertificateOrder.getOrderShare({
+            sweep: options?.fromSweep
+              ? {
+                  budget: Service.SWEEP_ORDER_BUDGET,
+                  maxPerWindow: Service.ORDER_MAX_PER_RUN,
+                }
+              : undefined,
+            onDemand: options?.onDemand
+              ? { projectId: domainToOrder.projectId?.toString() }
+              : undefined,
+          }),
+        });
+      },
+    });
+  }
+
+  /*
+   * Called by verify-cname - the domain's Check now - once it has found the
+   * domain's CNAME record: order the domain's free certificate now rather
+   * than at the next run of the order sweep, and say what happens to its
+   * certificate next. The same step as a status page domain's
+   * (CustomDomainOrders.orderOnceCnameIsVerified): one on-demand order per
+   * domain per 15 minutes, counted against the domain's project, under the
+   * name's lock and within the installation's Let's Encrypt allowance; never
+   * for an uploaded certificate; Issued only for a certificate that is there
+   * and has not expired. Never rejects.
+   */
+  @CaptureSpan()
+  public async orderCertOnceCnameIsVerified(
+    dashboardDomain: DashboardDomain,
+    options?: { waitInMs?: number | undefined },
+  ): Promise<CustomDomainVerificationResult> {
+    return await CustomDomainOrders.orderOnceCnameIsVerified({
+      domain: dashboardDomain,
+      waitInMs: options?.waitInMs,
+      recordAsOrdered: async (): Promise<void> => {
+        await this.recordCertificatesAsOrdered([dashboardDomain.id!]);
+      },
+      orderIfMissing: async (): Promise<CertificateOrderOutcome> => {
+        return await this.orderCertIfMissing(dashboardDomain, {
+          onDemand: true,
+          // The route found the record a moment ago.
+          cnameVerifiedJustNow: true,
+          renewIfExpired: true,
+        });
+      },
+    });
+  }
+
+  /*
+   * Where the certificates of these domains stand, for the Custom Domains
+   * page's Status column (CustomDomainCertificates): each one's expiry, and
+   * its last failed order while no order since has succeeded.
+   */
+  @CaptureSpan()
+  public async getCertificates(
+    domains: Array<DashboardDomain>,
+  ): Promise<Array<CustomDomainCertificate>> {
+    return await CustomDomainOrders.getCertificates(domains);
+  }
+
   /*
    * Reissue this domain's Let's Encrypt certificate because a customer asked
    * for it, rather than because it is close to expiring.
@@ -313,69 +551,72 @@ export class Service extends DatabaseService<DashboardDomain> {
     }
 
     /*
-     * A domain that never ordered a certificate has nothing to reissue - the
-     * dashboard shows "Order Free SSL" for it, which is the correct button.
+     * A domain that never ordered a certificate has nothing to reissue. Its
+     * first certificate is ordered on its own once the CNAME is verified, and
+     * the dashboard shows Reissue SSL only after that.
      */
     if (!dashboardDomain.isSslOrdered) {
       throw new BadDataException(
-        "No SSL certificate has been ordered for this domain yet. Please order one first.",
+        "No SSL certificate has been ordered for this domain yet. One is ordered automatically once its CNAME record is verified.",
       );
     }
 
-    /*
-     * Claim the cooldown before ordering anything, and claim it by putting the
-     * cooldown condition in the WHERE clause rather than in an `if` above the
-     * write. Two clicks that arrive together both read a row that is not
-     * cooling down; only the write can decide which one of them gets to spend
-     * the CA's allowance.
-     *
-     * The stamp goes down BEFORE the order and is never rolled back on
-     * failure. A rejected order still costs a validation attempt against the
-     * shared account, and a domain whose order fails is exactly the domain
-     * somebody presses again - so a failed attempt has to start the clock the
-     * same way a successful one does.
-     */
-    const claimedRowCount: number = await this.updateOneBy({
-      query: {
-        _id: domainId.toString(),
-        certificateReissueRequestedAt: QueryHelper.lessThanEqualToOrNull(
-          CertificateReissueUtil.getCooldownCutoff(now),
-        ),
-      },
-      data: {
-        certificateReissueRequestedAt: now,
+    await CertificateReissueOrder.reissue({
+      domain: dashboardDomain.fullDomain,
+      now: now,
+      lastReissueRequestedAt: dashboardDomain.certificateReissueRequestedAt,
+      claimCooldown: async (): Promise<number> => {
+        /*
+         * Claim the cooldown before ordering anything, and claim it by
+         * putting the cooldown condition in the WHERE clause rather than in
+         * an `if` above the write. Two clicks that arrive together both read
+         * a row that is not cooling down; only the write can decide which
+         * one of them gets to spend the CA's allowance.
+         */
+        return await this.updateOneBy({
+          query: {
+            _id: domainId.toString(),
+            certificateReissueRequestedAt: QueryHelper.lessThanEqualToOrNull(
+              CertificateReissueUtil.getCooldownCutoff(now),
+            ),
+          },
+          data: {
+            certificateReissueRequestedAt: now,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
       },
-      props: {
-        isRoot: true,
+      giveCooldownBack: async (): Promise<void> => {
+        await this.updateOneBy({
+          query: {
+            _id: domainId.toString(),
+            certificateReissueRequestedAt: now,
+          },
+          data: {
+            certificateReissueRequestedAt:
+              dashboardDomain.certificateReissueRequestedAt || null,
+          } as never,
+          props: {
+            isRoot: true,
+          },
+        });
       },
-    });
-
-    if (claimedRowCount === 0) {
-      if (dashboardDomain.certificateReissueRequestedAt) {
-        throw new TooManyRequestsException(
-          CertificateReissueUtil.getCooldownMessage(
-            dashboardDomain.certificateReissueRequestedAt,
-            now,
-          ),
+      order: async (
+        lock: CertificateOrderLockHandle,
+      ): Promise<CertificateOrderOutcome> => {
+        logger.debug(
+          "Reissuing SSL certificate for domain: " + dashboardDomain.fullDomain,
+          { fullDomain: dashboardDomain.fullDomain } as LogAttributes,
         );
-      }
-
-      /*
-       * The row matched the cooldown a moment ago and does not now: another
-       * request claimed it in between, or the domain was deleted. Either way
-       * the caller must not order.
-       */
-      throw new BadDataException(
-        "Could not start a certificate reissue for this domain. Please refresh the page and try again.",
-      );
-    }
-
-    logger.debug(
-      "Reissuing SSL certificate for domain: " + dashboardDomain.fullDomain,
-      { fullDomain: dashboardDomain.fullDomain } as LogAttributes,
-    );
 
-    await this.orderCert(dashboardDomain);
+        return await this.orderCert(dashboardDomain, {
+          reason: CertificateOrderReason.Reissue,
+          lock: lock,
+        });
+      },
+    });
   }
 
   @CaptureSpan()
@@ -387,6 +628,7 @@ export class Service extends DatabaseService<DashboardDomain> {
       },
       select: {
         _id: true,
+        fullDomain: true,
       },
       limit: LIMIT_MAX,
       skip: 0,
@@ -395,9 +637,20 @@ export class Service extends DatabaseService<DashboardDomain> {
       },
     });
 
-    for (const domain of domains) {
-      await this.updateSslProvisioningStatus(domain);
-    }
+    await ArrayUtil.forEachWithConcurrency(
+      domains,
+      Service.DOMAIN_CHECK_CONCURRENCY,
+      async (domain: DashboardDomain): Promise<void> => {
+        try {
+          await this.updateSslProvisioningStatus(domain);
+        } catch (err) {
+          // one unreachable domain must not end the sweep for the rest.
+          logger.error(err, {
+            fullDomain: domain.fullDomain,
+          } as LogAttributes);
+        }
+      },
+    );
   }
 
   private async isSSLProvisioned(
@@ -461,7 +714,32 @@ export class Service extends DatabaseService<DashboardDomain> {
   }
 
   @CaptureSpan()
-  public async isCnameValid(fullDomain: string): Promise<boolean> {
+  public async isCnameValid(
+    fullDomain: string,
+    options?: {
+      /*
+       * Whether the answer is written to the domain row: verified, or
+       * unverified, unordered and unprovisioned. The renewal run passes
+       * false - a DNS blip while it renews must not reset a domain whose
+       * certificate it keeps; the provisioning sweep still notices a
+       * record that is really gone.
+       */
+      recordResult?: boolean | undefined;
+    },
+  ): Promise<boolean> {
+    const recordResult: (cnameStatus: boolean) => Promise<void> = async (
+      cnameStatus: boolean,
+    ): Promise<void> => {
+      if (options?.recordResult === false) {
+        return;
+      }
+
+      await this.updateCnameStatusForDashboardDomain({
+        domain: fullDomain,
+        cnameStatus: cnameStatus,
+      });
+    };
+
     try {
       logger.debug("Checking for CNAME " + fullDomain, {
         fullDomain,
@@ -508,10 +786,7 @@ export class Service extends DatabaseService<DashboardDomain> {
         logger.debug(result, { fullDomain } as LogAttributes);
 
         if (result.isSuccess()) {
-          await this.updateCnameStatusForDashboardDomain({
-            domain: fullDomain,
-            cnameStatus: true,
-          });
+          await recordResult(true);
 
           return true;
         }
@@ -539,10 +814,7 @@ export class Service extends DatabaseService<DashboardDomain> {
         logger.debug(resultHttps, { fullDomain } as LogAttributes);
 
         if (resultHttps.isSuccess()) {
-          await this.updateCnameStatusForDashboardDomain({
-            domain: fullDomain,
-            cnameStatus: true,
-          });
+          await recordResult(true);
 
           return true;
         }
@@ -569,10 +841,7 @@ export class Service extends DatabaseService<DashboardDomain> {
               `No CNAME record found for ${fullDomain}. Expected record: ${DashboardCNameRecord}`,
               { fullDomain } as LogAttributes,
             );
-            await this.updateCnameStatusForDashboardDomain({
-              domain: fullDomain,
-              cnameStatus: false,
-            });
+            await recordResult(false);
             return false;
           }
 
@@ -586,10 +855,7 @@ export class Service extends DatabaseService<DashboardDomain> {
               { fullDomain } as LogAttributes,
             );
 
-            await this.updateCnameStatusForDashboardDomain({
-              domain: fullDomain,
-              cnameStatus: true,
-            });
+            await recordResult(true);
 
             return true;
           }
@@ -606,25 +872,33 @@ export class Service extends DatabaseService<DashboardDomain> {
         logger.debug(err, { fullDomain } as LogAttributes);
       }
 
-      await this.updateCnameStatusForDashboardDomain({
-        domain: fullDomain,
-        cnameStatus: false,
-      });
+      await recordResult(false);
 
       return false;
     } catch (err) {
       logger.debug("Failed checking for CNAME " + fullDomain);
       logger.debug(err);
 
-      await this.updateCnameStatusForDashboardDomain({
-        domain: fullDomain,
-        cnameStatus: false,
-      });
+      await recordResult(false);
 
       return false;
     }
   }
 
+  /*
+   * Record whether this domain's certificate is being served yet.
+   *
+   * Unlike the status page sweep this never orders a certificate. A probe
+   * that fails right after an order is usually racing nginx, which writes
+   * new certificates to disk every 15 minutes, and re-ordering then spends a
+   * second order on the same name. The two cases that do need an order have
+   * capped sweeps of their own: checkOrderStatus re-orders a certificate that
+   * has gone missing, and the renewal run renews one that is about to expire.
+   *
+   * When the probe fails the CNAME is checked again, so a domain whose record
+   * was removed goes back to asking for it. The result is written only when
+   * it changes: this sweep visits every ordered domain every 15 minutes.
+   */
   @CaptureSpan()
   public async updateSslProvisioningStatus(
     domain: DashboardDomain,
@@ -641,6 +915,7 @@ export class Service extends DatabaseService<DashboardDomain> {
         _id: true,
         fullDomain: true,
         cnameVerificationToken: true,
+        isSslProvisioned: true,
       },
       props: {
         isRoot: true,
@@ -652,56 +927,88 @@ export class Service extends DatabaseService<DashboardDomain> {
     }
 
     logger.debug(
-      `DashboardCerts:RemoveCerts - Checking CNAME ${dashboardDomain.fullDomain}`,
+      `DashboardCerts:CheckSslProvisioningStatus - Checking ${dashboardDomain.fullDomain}`,
       { fullDomain: dashboardDomain.fullDomain } as LogAttributes,
     );
 
-    const isValid: boolean = await this.isSSLProvisioned(
+    const isProvisioned: boolean = await this.isSSLProvisioned(
       dashboardDomain.fullDomain!,
       dashboardDomain.cnameVerificationToken!,
     );
 
-    if (!isValid) {
-      const isCnameValid: boolean = await this.isCnameValid(
-        dashboardDomain.fullDomain!,
-      );
+    if (!isProvisioned) {
+      // isCnameValid also records the result on the domain.
+      await this.isCnameValid(dashboardDomain.fullDomain!);
+    }
 
+    if (Boolean(dashboardDomain.isSslProvisioned) !== isProvisioned) {
       await this.updateOneById({
         id: dashboardDomain.id!,
         data: {
-          isSslProvisioned: false,
+          isSslProvisioned: isProvisioned,
         },
         props: {
           isRoot: true,
         },
       });
+    }
+  }
 
-      if (isCnameValid) {
-        try {
-          await this.orderCert(dashboardDomain);
-        } catch (err) {
-          logger.error(
-            "Cannot order cert for domain: " + dashboardDomain.fullDomain,
-            { fullDomain: dashboardDomain.fullDomain } as LogAttributes,
-          );
-          logger.error(err, {
-            fullDomain: dashboardDomain.fullDomain,
-          } as LogAttributes);
-        }
-      }
-    } else {
-      await this.updateOneById({
-        id: dashboardDomain.id!,
-        data: {
-          isSslProvisioned: true,
+  /*
+   * Order the first certificate of each of these domains, whose CNAME is
+   * verified: at most ORDER_MAX_PER_RUN orders in a window between the
+   * sweeps that order, picked afresh every run; a domain that has a
+   * certificate already is only recorded as ordered, and one whose orders
+   * keep failing waits longer after each failure
+   * (CustomDomainOrders.orderForVerifiedDomainsWithoutOne, which the status
+   * page sweeps share).
+   */
+  private async orderCertsForVerifiedDomainsWithoutOne(
+    domains: Array<DashboardDomain>,
+    options?: {
+      // Every one of them was verified a moment ago, by this caller.
+      cnameVerifiedJustNow?: boolean | undefined;
+    },
+  ): Promise<void> {
+    await CustomDomainOrders.orderForVerifiedDomainsWithoutOne<DashboardDomain>(
+      {
+        domains: domains,
+        maxPerRun: Service.ORDER_MAX_PER_RUN,
+        recordAsOrdered: async (domainIds: Array<ObjectID>): Promise<void> => {
+          await this.recordCertificatesAsOrdered(domainIds);
         },
-        props: {
-          isRoot: true,
+        orderIfMissing: async (
+          domain: DashboardDomain,
+        ): Promise<CertificateOrderOutcome> => {
+          return await this.orderCertIfMissing(domain, {
+            fromSweep: true,
+            cnameVerifiedJustNow: options?.cnameVerifiedJustNow,
+          });
         },
-      });
-    }
+      },
+    );
   }
 
+  /*
+   * Order the first certificate for domains whose CNAME is verified.
+   *
+   * Only verified domains: verifyCnameWhoseCnameisNotVerified checks the
+   * rest - and orders each one's certificate the moment it is verified, so
+   * this sweep is the fallback for the rest - and ordering one whose record
+   * is not in place would only fail the CNAME check again.
+   *
+   * A domain that already has a certificate is only recorded as ordered -
+   * all of them in one write. That is the domain whose CNAME check failed
+   * for a moment - which marks it unordered - and then passed again.
+   * Ordering for it would spend an order on a duplicate (Let's Encrypt
+   * allows five a week per name), and if its certificate is due, the renewal
+   * run orders it in this same tick. The rest are ordered through
+   * orderCertIfMissing, at most ORDER_MAX_PER_RUN in a window between the
+   * sweeps that order, picked afresh every run
+   * (GreenlockUtil.pickForThisRun), so a domain whose order keeps failing
+   * cannot hold a slot the others are waiting for; it also waits longer
+   * after each failure in a row (CertificateOrderFailures).
+   */
   @CaptureSpan()
   public async orderSSLForDomainsWhichAreNotOrderedYet(): Promise<void> {
     return Telemetry.startActiveSpan<Promise<void>>({
@@ -709,34 +1016,25 @@ export class Service extends DatabaseService<DashboardDomain> {
       options: { attributes: {} },
       fn: async (span: Span): Promise<void> => {
         try {
-          const domains: Array<DashboardDomain> = await this.findBy({
+          const domains: Array<DashboardDomain> = await this.findAllBy({
             query: {
+              isCnameVerified: true,
               isSslOrdered: false,
               isCustomCertificate: false,
             },
             select: {
               _id: true,
               fullDomain: true,
+              isSslOrdered: true,
+              isCustomCertificate: true,
             },
-            limit: LIMIT_MAX,
             skip: 0,
             props: {
               isRoot: true,
             },
           });
 
-          for (const domain of domains) {
-            try {
-              logger.debug("Ordering SSL for domain: " + domain.fullDomain, {
-                fullDomain: domain.fullDomain,
-              } as LogAttributes);
-              await this.orderCert(domain);
-            } catch (e) {
-              logger.error(e, {
-                fullDomain: domain.fullDomain,
-              } as LogAttributes);
-            }
-          }
+          await this.orderCertsForVerifiedDomainsWithoutOne(domains);
 
           Telemetry.endSpan(span);
         } catch (err) {
@@ -751,6 +1049,15 @@ export class Service extends DatabaseService<DashboardDomain> {
     });
   }
 
+  /*
+   * Check every domain whose CNAME is not verified yet, at most
+   * DOMAIN_CHECK_CONCURRENCY at a time, and order the certificate of each
+   * one that is verified now straight away rather than at the next run of
+   * the order sweep - as the status page sweep does. So a domain is served
+   * on its own certificate within about 15 minutes of its record going live
+   * - nginx writes new certificates every 15 minutes - whether or not
+   * anyone clicked Check now.
+   */
   @CaptureSpan()
   public async verifyCnameWhoseCnameisNotVerified(): Promise<void> {
     const domains: Array<DashboardDomain> = await this.findBy({
@@ -760,6 +1067,8 @@ export class Service extends DatabaseService<DashboardDomain> {
       select: {
         _id: true,
         fullDomain: true,
+        isCustomCertificate: true,
+        isSslOrdered: true,
       },
       limit: LIMIT_MAX,
       skip: 0,
@@ -768,20 +1077,97 @@ export class Service extends DatabaseService<DashboardDomain> {
       },
     });
 
-    for (const domain of domains) {
-      try {
-        await this.isCnameValid(domain.fullDomain as string);
-      } catch (e) {
-        logger.error(e, { fullDomain: domain.fullDomain } as LogAttributes);
-      }
+    const verifiedNow: Array<DashboardDomain> = [];
+
+    await ArrayUtil.forEachWithConcurrency(
+      domains,
+      Service.DOMAIN_CHECK_CONCURRENCY,
+      async (domain: DashboardDomain): Promise<void> => {
+        try {
+          // isCnameValid also records the result on the domain.
+          const isVerified: boolean = await this.isCnameValid(
+            domain.fullDomain as string,
+          );
+
+          if (isVerified && !domain.isCustomCertificate) {
+            verifiedNow.push(domain);
+          }
+        } catch (e) {
+          logger.error(e, { fullDomain: domain.fullDomain } as LogAttributes);
+        }
+      },
+    );
+
+    try {
+      /*
+       * Each was verified by this run a moment ago, so the orders do not
+       * check the CNAME again.
+       */
+      await this.orderCertsForVerifiedDomainsWithoutOne(verifiedNow, {
+        cnameVerifiedJustNow: true,
+      });
+    } catch (e) {
+      // The order sweep orders them on its next run.
+      logger.error(e);
+    }
+  }
+
+  /*
+   * Which of these certificate domains are dashboard domains. The renewal
+   * run renews - and removes - only the certificates this returns, so the
+   * certificates of status page domains and of the primary host, which share
+   * the AcmeCertificate table, are never mistaken for dashboard domains
+   * whose CNAME stopped validating.
+   *
+   * A domain serving a certificate its owner uploaded still claims its
+   * Let's Encrypt certificate, which stays renewed meanwhile: nginx serves
+   * the uploaded one for that name (AcmeWriteCertificates leaves it alone),
+   * and switching back is then instant, on every nginx replica, including
+   * ones that start with an empty certificate directory.
+   */
+  @CaptureSpan()
+  public async getOwnedDomains(domains: Array<string>): Promise<Array<string>> {
+    if (domains.length === 0) {
+      return [];
     }
+
+    const dashboardDomains: Array<DashboardDomain> = await this.findBy({
+      query: {
+        fullDomain: QueryHelper.any(domains),
+      },
+      select: {
+        fullDomain: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    return dashboardDomains
+      .map((dashboardDomain: DashboardDomain) => {
+        return dashboardDomain.fullDomain || "";
+      })
+      .filter((fullDomain: string) => {
+        return fullDomain.length > 0;
+      });
   }
 
   @CaptureSpan()
   public async renewCertsWhichAreExpiringSoon(): Promise<void> {
     await GreenlockUtil.renewAllCertsWhichAreExpiringSoon({
+      maxPerRun: Service.ORDER_MAX_PER_RUN,
+      getOwnedDomains: async (domains: Array<string>) => {
+        return await this.getOwnedDomains(domains);
+      },
+      /*
+       * A check that writes nothing: a DNS blip while a certificate is
+       * renewed must not mark the domain unverified and unordered while its
+       * certificate keeps serving.
+       */
       validateCname: async (fullDomain: string) => {
-        return await this.isCnameValid(fullDomain);
+        return await this.isCnameValid(fullDomain, { recordResult: false });
       },
       notifyDomainRemoved: async (domain: string) => {
         await this.updateOneBy({
@@ -804,9 +1190,22 @@ export class Service extends DatabaseService<DashboardDomain> {
     });
   }
 
+  /*
+   * Re-order the certificate of a domain that says it has one ordered but
+   * whose certificate is gone, at most ORDER_MAX_PER_RUN per run.
+   *
+   * This is how dashboard domains recover from the status page renewal job,
+   * which until it learned to check ownership deleted every dashboard
+   * certificate that came due for renewal while the domain kept saying
+   * "ordered" - nginx then served the stale copy on disk until it expired.
+   * Domains are picked afresh every run, for the same reason as in the
+   * first-order sweep above, and re-ordered through orderCertIfMissing, so a
+   * re-order never races another order for the same name and shares the
+   * sweeps' budget.
+   */
   @CaptureSpan()
   public async checkOrderStatus(): Promise<void> {
-    const domains: Array<DashboardDomain> = await this.findBy({
+    const domains: Array<DashboardDomain> = await this.findAllBy({
       query: {
         isSslOrdered: true,
         isCustomCertificate: false,
@@ -814,42 +1213,69 @@ export class Service extends DatabaseService<DashboardDomain> {
       select: {
         _id: true,
         fullDomain: true,
-        cnameVerificationToken: true,
+        isCustomCertificate: true,
       },
-      limit: LIMIT_MAX,
       skip: 0,
       props: {
         isRoot: true,
       },
     });
 
-    for (const domain of domains) {
-      if (!domain.fullDomain) {
-        continue;
-      }
+    const fullDomains: Array<string> = domains
+      .map((domain: DashboardDomain) => {
+        return domain.fullDomain || "";
+      })
+      .filter((fullDomain: string) => {
+        return fullDomain.length > 0;
+      });
 
-      const acmeCert: AcmeCertificate | null =
-        await AcmeCertificateService.findOneBy({
-          query: {
-            domain: domain.fullDomain,
-          },
-          select: {
-            _id: true,
-          },
-          props: {
-            isRoot: true,
-          },
-        });
+    if (fullDomains.length === 0) {
+      return;
+    }
 
-      if (!acmeCert) {
-        try {
-          await this.orderCert(domain);
-        } catch (err) {
-          logger.error("Cannot order cert for domain: " + domain.fullDomain, {
-            fullDomain: domain.fullDomain,
-          } as LogAttributes);
-          logger.error(err, { fullDomain: domain.fullDomain } as LogAttributes);
-        }
+    const certificates: Map<string, AcmeCertificate> =
+      await GreenlockUtil.findCertificatesByDomain(
+        fullDomains.map((fullDomain: string) => {
+          return CertificateOrder.normalizeDomain(fullDomain);
+        }),
+      );
+
+    const now: Date = OneUptimeDate.getCurrentDate();
+
+    const domainsWithoutCertificate: Array<DashboardDomain> =
+      await CertificateOrderFailures.withoutThoseWaitingToRetry({
+        items: domains.filter((domain: DashboardDomain) => {
+          return (
+            Boolean(domain.id) &&
+            Boolean(domain.fullDomain) &&
+            !certificates.has(
+              CertificateOrder.normalizeDomain(domain.fullDomain as string),
+            )
+          );
+        }),
+        getDomain: (domain: DashboardDomain): string => {
+          return domain.fullDomain as string;
+        },
+        now: now,
+      });
+
+    const batch: Array<DashboardDomain> = GreenlockUtil.pickForThisRun({
+      items: domainsWithoutCertificate,
+      max: Service.ORDER_MAX_PER_RUN,
+      now: now,
+      getKey: (domain: DashboardDomain): string => {
+        return domain.fullDomain || "";
+      },
+    });
+
+    for (const domain of batch) {
+      try {
+        await this.orderCertIfMissing(domain, { fromSweep: true });
+      } catch (err) {
+        logger.error("Cannot order cert for domain: " + domain.fullDomain, {
+          fullDomain: domain.fullDomain,
+        } as LogAttributes);
+        logger.error(err, { fullDomain: domain.fullDomain } as LogAttributes);
       }
     }
   }