@oneuptime/common 14.0.13 → 14.0.14

package/Server/Services/StatusPageDomainService.ts

@@ -3,6 +3,14 @@ import DeleteBy from "../Types/Database/DeleteBy";
 import UpdateBy from "../Types/Database/UpdateBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import GreenlockUtil from "../Utils/Greenlock/Greenlock";
+import CertificateOrder, {
+  CertificateOrderOutcome,
+} from "../Utils/Greenlock/CertificateOrder";
+import CustomDomainOrders from "../Utils/Greenlock/CustomDomainOrders";
+import { CertificateOrderLockHandle } from "../Utils/Greenlock/CertificateOrderLock";
+import { CertificateOrderReason } from "../Utils/Greenlock/CertificateOrderBudget";
+import CertificateOrderFailures from "../Utils/Greenlock/CertificateOrderFailures";
+import CertificateReissueOrder from "../Utils/Greenlock/CertificateReissueOrder";
 import logger, { LogAttributes } from "../Utils/Logger";
 import DatabaseService from "./DatabaseService";
 import DomainService from "./DomainService";
@@ -15,7 +23,6 @@ import ObjectID from "../../Types/ObjectID";
 import AcmeCertificate from "../../Models/DatabaseModels/AcmeCertificate";
 import DomainModel from "../../Models/DatabaseModels/Domain";
 import StatusPageDomain from "../../Models/DatabaseModels/StatusPageDomain";
-import AcmeCertificateService from "./AcmeCertificateService";
 import Telemetry, { Span } from "../Utils/Telemetry";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { StatusPageCNameRecord } from "../EnvironmentConfig";
@@ -23,8 +30,9 @@ import Domain from "../Types/Domain";
 import ArrayUtil from "../../Utils/Array";
 import OneUptimeDate from "../../Types/Date";
 import CertificateReissueUtil from "../../Utils/CertificateReissue";
-import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
 import QueryHelper from "../Types/Database/QueryHelper";
+import { CustomDomainVerificationResult } from "../../Types/CustomDomain/CustomDomainVerification";
+import { CustomDomainCertificate } from "../../Types/CustomDomain/CustomDomainCertificates";
 
 const STATUS_PAGE_DOMAIN_EGRESS_LABEL: string = "Status page domain";
 
@@ -38,6 +46,37 @@ export class Service extends DatabaseService<StatusPageDomain> {
    */
   public static readonly SSL_PROVISIONING_CHECK_CONCURRENCY: number = 10;
 
+  /*
+   * How many certificates the sweeps that order them may order in one
+   * 15-minute window, between them: the order for the domains the CNAME
+   * sweep has just verified, the order sweep for every verified domain still
+   * without a certificate, and the re-order of a certificate that has gone
+   * missing. They draw from one budget (CertificateOrder.takeOrderSlot,
+   * SWEEP_ORDER_BUDGET), and each picks its batch afresh every run
+   * (GreenlockUtil.pickForThisRun), so a domain whose order keeps failing -
+   * a CAA record that leaves Let's Encrypt out, a name it has paused -
+   * cannot hold a slot the others are waiting for, and waits longer after
+   * each failed order (CertificateOrderFailures). Check now is not counted
+   * here: it has a window of its own per domain
+   * (CertificateOrder.claimOnDemandOrder) and the on-demand budget.
+   *
+   * Every order spends from the one Let's Encrypt account the whole
+   * installation shares, 300 new orders per three hours, renewals included.
+   * Whatever these caps allow, no order is placed beyond the installation's
+   * own budget for that account (CertificateOrderBudget), which every order
+   * of every kind draws from. In a steady state these sweeps order a few
+   * certificates a day: a domain whose owner clicks Check now is ordered by
+   * that request. The cap is for a backlog, such as the domains the order
+   * sweep used to retry every 15 minutes without any limit, which is now
+   * worked off a few domains a run the way the dashboard sweeps
+   * (DashboardDomainService.ORDER_MAX_PER_RUN) and renewals
+   * (GreenlockUtil.RENEW_MAX_PER_RUN) are.
+   */
+  public static readonly ORDER_MAX_PER_RUN: number = 5;
+
+  // The budget the ordering sweeps share (CertificateOrder.takeOrderSlot).
+  public static readonly SWEEP_ORDER_BUDGET: string = "StatusPageDomainSweeps";
+
   public constructor() {
     super(StatusPageDomain);
   }
@@ -181,9 +220,82 @@ export class Service extends DatabaseService<StatusPageDomain> {
     await GreenlockUtil.removeDomain(domain);
   }
 
+  /*
+   * The domain with its full name and whether it serves an uploaded
+   * certificate, read once when the caller did not have them both.
+   */
+  private async getDomainToOrder(
+    statusPageDomain: StatusPageDomain,
+  ): Promise<StatusPageDomain> {
+    if (!statusPageDomain.id) {
+      throw new BadDataException(
+        "Domain ID is required to order a certificate",
+      );
+    }
+
+    if (
+      statusPageDomain.fullDomain &&
+      statusPageDomain.isCustomCertificate !== undefined
+    ) {
+      return statusPageDomain;
+    }
+
+    const fetchedStatusPageDomain: StatusPageDomain | null =
+      await this.findOneBy({
+        query: {
+          _id: statusPageDomain.id.toString(),
+        },
+        select: {
+          _id: true,
+          fullDomain: true,
+          isCustomCertificate: true,
+          projectId: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+    if (!fetchedStatusPageDomain) {
+      throw new BadDataException("DomainModel not found");
+    }
+
+    if (!fetchedStatusPageDomain.fullDomain) {
+      throw new BadDataException(
+        "Unable to order certificate because domain is null",
+      );
+    }
+
+    return fetchedStatusPageDomain;
+  }
+
+  /*
+   * Order this domain's certificate and record it: a first certificate
+   * (the default), or a reissue. GreenlockUtil.orderCert places it under the
+   * name's order lock - the one handed in, or its own - and within the
+   * installation's Let's Encrypt allowance; the outcome says whether it
+   * ordered.
+   */
   @CaptureSpan()
-  public async orderCert(statusPageDomain: StatusPageDomain): Promise<void> {
-    return Telemetry.startActiveSpan<Promise<void>>({
+  public async orderCert(
+    statusPageDomain: StatusPageDomain,
+    options?: {
+      reason?: CertificateOrderReason | undefined;
+      // The name's order lock, when the caller holds it.
+      lock?: CertificateOrderLockHandle | undefined;
+      /*
+       * The caller has just found the domain's CNAME record: the order
+       * does not check it again (see GreenlockUtil.orderCert).
+       */
+      cnameVerifiedJustNow?: boolean | undefined;
+      /*
+       * The caller's share of orders (CertificateOrder.getOrderShare),
+       * asked once the CNAME check has passed.
+       */
+      mayOrder?: (() => Promise<boolean>) | undefined;
+    },
+  ): Promise<CertificateOrderOutcome> {
+    return Telemetry.startActiveSpan<Promise<CertificateOrderOutcome>>({
       name: "StatusPageDomainService.orderCert",
       options: {
         attributes: {
@@ -191,65 +303,58 @@ export class Service extends DatabaseService<StatusPageDomain> {
           _id: statusPageDomain.id?.toString(),
         },
       },
-      fn: async (span: Span): Promise<void> => {
+      fn: async (span: Span): Promise<CertificateOrderOutcome> => {
         try {
-          if (!statusPageDomain.fullDomain) {
-            const fetchedStatusPageDomain: StatusPageDomain | null =
-              await this.findOneBy({
-                query: {
-                  _id: statusPageDomain.id!.toString(),
-                },
-                select: {
-                  _id: true,
-                  fullDomain: true,
-                },
-                props: {
-                  isRoot: true,
-                },
-              });
-
-            if (!fetchedStatusPageDomain) {
-              throw new BadDataException("DomainModel not found");
-            }
-
-            statusPageDomain = fetchedStatusPageDomain;
-          }
+          const domainToOrder: StatusPageDomain =
+            await this.getDomainToOrder(statusPageDomain);
 
-          if (!statusPageDomain.fullDomain) {
-            throw new BadDataException(
-              "Unable to order certificate because domain is null",
-            );
-          }
-
-          logger.debug(
-            "Ordering SSL for domain: " + statusPageDomain.fullDomain,
-            { fullDomain: statusPageDomain.fullDomain } as LogAttributes,
-          );
+          logger.debug("Ordering SSL for domain: " + domainToOrder.fullDomain, {
+            fullDomain: domainToOrder.fullDomain,
+          } as LogAttributes);
 
-          await GreenlockUtil.orderCert({
-            domain: statusPageDomain.fullDomain as string,
-            validateCname: async (fullDomain: string) => {
-              return await this.isCnameValid(fullDomain);
-            },
-          });
+          const outcome: CertificateOrderOutcome =
+            await GreenlockUtil.orderCert({
+              domain: domainToOrder.fullDomain as string,
+              reason:
+                options?.reason || CertificateOrderReason.FirstCertificate,
+              validateCname: options?.cnameVerifiedJustNow
+                ? null
+                : async (fullDomain: string) => {
+                    return await this.isCnameValid(fullDomain);
+                  },
+              lock: options?.lock,
+              mayOrder: options?.mayOrder,
+            });
 
-          logger.debug(
-            "SSL ordered for domain: " + statusPageDomain.fullDomain,
-            { fullDomain: statusPageDomain.fullDomain } as LogAttributes,
-          );
+          if (outcome === CertificateOrderOutcome.Ordered) {
+            logger.debug(
+              "SSL ordered for domain: " + domainToOrder.fullDomain,
+              { fullDomain: domainToOrder.fullDomain } as LogAttributes,
+            );
 
-          // update the order.
-          await this.updateOneById({
-            id: statusPageDomain.id!,
-            data: {
-              isSslOrdered: true,
-            },
-            props: {
-              isRoot: true,
-            },
-          });
+            /*
+             * Verified as well as ordered: Let's Encrypt has just fetched
+             * its challenge from this domain, so the domain reaches us. A
+             * CNAME sweep that saw a DNS blip while the order ran - and
+             * marked the domain unverified and unordered - would otherwise
+             * leave it "ordered" but "waiting for DNS", with a certificate
+             * the dialog had just called verified.
+             */
+            await this.updateOneById({
+              id: domainToOrder.id!,
+              data: {
+                isCnameVerified: true,
+                isSslOrdered: true,
+              },
+              props: {
+                isRoot: true,
+              },
+            });
+          }
 
           Telemetry.endSpan(span);
+
+          return outcome;
         } catch (err) {
           Telemetry.recordExceptionMarkSpanAsErrorAndEndSpan({
             span,
@@ -262,6 +367,42 @@ export class Service extends DatabaseService<StatusPageDomain> {
     });
   }
 
+  /*
+   * Records these domains as ordered: they have a certificate, ordered
+   * before, so nothing is ordered. One write for all of them, and only on
+   * domains that are still verified - a CNAME check that failed meanwhile
+   * marked a domain unverified and unordered, and marking it ordered again
+   * would leave it ordered but "waiting for DNS". The next CNAME check that
+   * passes records it again.
+   */
+  private async recordCertificatesAsOrdered(
+    domainIds: Array<ObjectID>,
+  ): Promise<void> {
+    if (domainIds.length === 0) {
+      return;
+    }
+
+    await this.updateBy({
+      query: {
+        _id: QueryHelper.any(
+          domainIds.map((id: ObjectID) => {
+            return id.toString();
+          }),
+        ),
+        isCnameVerified: true,
+        isSslOrdered: false,
+      },
+      data: {
+        isSslOrdered: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+
   /*
    * Reissue this domain's Let's Encrypt certificate because a customer asked
    * for it, rather than because it is close to expiring.
@@ -324,69 +465,215 @@ export class Service extends DatabaseService<StatusPageDomain> {
     }
 
     /*
-     * A domain that never ordered a certificate has nothing to reissue - the
-     * dashboard shows "Order Free SSL" for it, which is the correct button.
+     * A domain that never ordered a certificate has nothing to reissue. Its
+     * first certificate is ordered on its own once the CNAME is verified, and
+     * the dashboard shows Reissue SSL only after that.
      */
     if (!statusPageDomain.isSslOrdered) {
       throw new BadDataException(
-        "No SSL certificate has been ordered for this domain yet. Please order one first.",
+        "No SSL certificate has been ordered for this domain yet. One is ordered automatically once its CNAME record is verified.",
       );
     }
 
-    /*
-     * Claim the cooldown before ordering anything, and claim it by putting the
-     * cooldown condition in the WHERE clause rather than in an `if` above the
-     * write. Two clicks that arrive together both read a row that is not
-     * cooling down; only the write can decide which one of them gets to spend
-     * the CA's allowance.
-     *
-     * The stamp goes down BEFORE the order and is never rolled back on
-     * failure. A rejected order still costs a validation attempt against the
-     * shared account, and a domain whose order fails is exactly the domain
-     * somebody presses again - so a failed attempt has to start the clock the
-     * same way a successful one does.
-     */
-    const claimedRowCount: number = await this.updateOneBy({
-      query: {
-        _id: domainId.toString(),
-        certificateReissueRequestedAt: QueryHelper.lessThanEqualToOrNull(
-          CertificateReissueUtil.getCooldownCutoff(now),
-        ),
+    await CertificateReissueOrder.reissue({
+      domain: statusPageDomain.fullDomain,
+      now: now,
+      lastReissueRequestedAt: statusPageDomain.certificateReissueRequestedAt,
+      claimCooldown: async (): Promise<number> => {
+        /*
+         * Claim the cooldown before ordering anything, and claim it by
+         * putting the cooldown condition in the WHERE clause rather than in
+         * an `if` above the write. Two clicks that arrive together both read
+         * a row that is not cooling down; only the write can decide which
+         * one of them gets to spend the CA's allowance.
+         */
+        return await this.updateOneBy({
+          query: {
+            _id: domainId.toString(),
+            certificateReissueRequestedAt: QueryHelper.lessThanEqualToOrNull(
+              CertificateReissueUtil.getCooldownCutoff(now),
+            ),
+          },
+          data: {
+            certificateReissueRequestedAt: now,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
       },
-      data: {
-        certificateReissueRequestedAt: now,
+      giveCooldownBack: async (): Promise<void> => {
+        await this.updateOneBy({
+          query: {
+            _id: domainId.toString(),
+            certificateReissueRequestedAt: now,
+          },
+          data: {
+            certificateReissueRequestedAt:
+              statusPageDomain.certificateReissueRequestedAt || null,
+          } as never,
+          props: {
+            isRoot: true,
+          },
+        });
       },
-      props: {
-        isRoot: true,
+      order: async (
+        lock: CertificateOrderLockHandle,
+      ): Promise<CertificateOrderOutcome> => {
+        logger.debug(
+          "Reissuing SSL certificate for domain: " +
+            statusPageDomain.fullDomain,
+          { fullDomain: statusPageDomain.fullDomain } as LogAttributes,
+        );
+
+        return await this.orderCert(statusPageDomain, {
+          reason: CertificateOrderReason.Reissue,
+          lock: lock,
+        });
       },
     });
+  }
 
-    if (claimedRowCount === 0) {
-      if (statusPageDomain.certificateReissueRequestedAt) {
-        throw new TooManyRequestsException(
-          CertificateReissueUtil.getCooldownMessage(
-            statusPageDomain.certificateReissueRequestedAt,
-            now,
-          ),
-        );
-      }
-
+  /*
+   * Order this domain's free certificate unless it has one already, or one
+   * is being ordered for it right now (CertificateOrder.orderIfMissing): the
+   * one way a first certificate is ordered, whether by Check now, by the
+   * order sweeps or by the order-ssl API.
+   *
+   * Never for a domain that serves a certificate its owner uploaded. Throws
+   * what the order throws, so a caller can show why.
+   */
+  @CaptureSpan()
+  public async orderCertIfMissing(
+    statusPageDomain: StatusPageDomain,
+    options?: {
+      // An order of the sweeps: it draws from their shared window budget.
+      fromSweep?: boolean | undefined;
       /*
-       * The row matched the cooldown a moment ago and does not now: another
-       * request claimed it in between, or the domain was deleted. Either way
-       * the caller must not order.
+       * An order somebody asked for (Check now, the order API): it draws
+       * from the on-demand budget (CertificateOrder.ON_DEMAND_ORDER_BUDGET).
        */
+      onDemand?: boolean | undefined;
+      // The caller has just found the domain's CNAME record.
+      cnameVerifiedJustNow?: boolean | undefined;
+      // Order for an expired certificate too (Check now).
+      renewIfExpired?: boolean | undefined;
+    },
+  ): Promise<CertificateOrderOutcome> {
+    const domainToOrder: StatusPageDomain =
+      await this.getDomainToOrder(statusPageDomain);
+
+    // The sweeps never order for one either: it serves the upload.
+    if (domainToOrder.isCustomCertificate) {
       throw new BadDataException(
-        "Could not start a certificate reissue for this domain. Please refresh the page and try again.",
+        "This domain uses a certificate you uploaded, so there is no free SSL certificate to order for it.",
       );
     }
 
-    logger.debug(
-      "Reissuing SSL certificate for domain: " + statusPageDomain.fullDomain,
-      { fullDomain: statusPageDomain.fullDomain } as LogAttributes,
+    const domainId: ObjectID = domainToOrder.id!;
+
+    return await CertificateOrder.orderIfMissing({
+      domain: domainToOrder.fullDomain as string,
+      renewIfExpired: options?.renewIfExpired,
+      recordAsOrdered: async (): Promise<void> => {
+        await this.recordCertificatesAsOrdered([domainId]);
+      },
+      order: async (
+        lock: CertificateOrderLockHandle,
+      ): Promise<CertificateOrderOutcome> => {
+        return await this.orderCert(domainToOrder, {
+          reason: CertificateOrderReason.FirstCertificate,
+          lock: lock,
+          cnameVerifiedJustNow: options?.cnameVerifiedJustNow,
+          mayOrder: CertificateOrder.getOrderShare({
+            sweep: options?.fromSweep
+              ? {
+                  budget: Service.SWEEP_ORDER_BUDGET,
+                  maxPerWindow: Service.ORDER_MAX_PER_RUN,
+                }
+              : undefined,
+            onDemand: options?.onDemand
+              ? { projectId: domainToOrder.projectId?.toString() }
+              : undefined,
+          }),
+        });
+      },
+    });
+  }
+
+  /*
+   * Called by verify-cname once it has found the domain's CNAME record:
+   * order the domain's free certificate now rather than at the next run of
+   * the order sweep, and say what happens to its certificate next
+   * (CustomDomainOrders.orderOnceCnameIsVerified, which dashboard domains
+   * share). Never rejects.
+   */
+  @CaptureSpan()
+  public async orderCertOnceCnameIsVerified(
+    statusPageDomain: StatusPageDomain,
+    options?: { waitInMs?: number | undefined },
+  ): Promise<CustomDomainVerificationResult> {
+    return await CustomDomainOrders.orderOnceCnameIsVerified({
+      domain: statusPageDomain,
+      waitInMs: options?.waitInMs,
+      recordAsOrdered: async (): Promise<void> => {
+        await this.recordCertificatesAsOrdered([statusPageDomain.id!]);
+      },
+      orderIfMissing: async (): Promise<CertificateOrderOutcome> => {
+        return await this.orderCertIfMissing(statusPageDomain, {
+          onDemand: true,
+          // The route found the record a moment ago.
+          cnameVerifiedJustNow: true,
+          renewIfExpired: true,
+        });
+      },
+    });
+  }
+
+  /*
+   * Order the first certificate of each of these domains, whose CNAME is
+   * verified: at most ORDER_MAX_PER_RUN orders, picked afresh every run; a
+   * domain that has a certificate already is only recorded as ordered, and
+   * one whose orders keep failing waits longer after each failure
+   * (CustomDomainOrders.orderForVerifiedDomainsWithoutOne, which the
+   * dashboard sweeps share).
+   */
+  private async orderCertsForVerifiedDomainsWithoutOne(
+    domains: Array<StatusPageDomain>,
+    options?: {
+      // Every one of them was verified a moment ago, by this caller.
+      cnameVerifiedJustNow?: boolean | undefined;
+    },
+  ): Promise<void> {
+    await CustomDomainOrders.orderForVerifiedDomainsWithoutOne<StatusPageDomain>(
+      {
+        domains: domains,
+        maxPerRun: Service.ORDER_MAX_PER_RUN,
+        recordAsOrdered: async (domainIds: Array<ObjectID>): Promise<void> => {
+          await this.recordCertificatesAsOrdered(domainIds);
+        },
+        orderIfMissing: async (
+          domain: StatusPageDomain,
+        ): Promise<CertificateOrderOutcome> => {
+          return await this.orderCertIfMissing(domain, {
+            fromSweep: true,
+            cnameVerifiedJustNow: options?.cnameVerifiedJustNow,
+          });
+        },
+      },
     );
+  }
 
-    await this.orderCert(statusPageDomain);
+  /*
+   * Where the certificates of these domains stand, for the Custom Domains
+   * page's Status column (CustomDomainCertificates): each one's expiry, and
+   * its last failed order while no order since has succeeded.
+   */
+  @CaptureSpan()
+  public async getCertificates(
+    domains: Array<StatusPageDomain>,
+  ): Promise<Array<CustomDomainCertificate>> {
+    return await CustomDomainOrders.getCertificates(domains);
   }
 
   @CaptureSpan()
@@ -491,7 +778,32 @@ export class Service extends DatabaseService<StatusPageDomain> {
   }
 
   @CaptureSpan()
-  public async isCnameValid(fullDomain: string): Promise<boolean> {
+  public async isCnameValid(
+    fullDomain: string,
+    options?: {
+      /*
+       * Whether the answer is written to the domain row: verified, or
+       * unverified, unordered and unprovisioned. The renewal run passes
+       * false - a DNS blip while it renews must not reset a domain whose
+       * certificate it keeps; the provisioning sweep still notices a
+       * record that is really gone.
+       */
+      recordResult?: boolean | undefined;
+    },
+  ): Promise<boolean> {
+    const recordResult: (cnameStatus: boolean) => Promise<void> = async (
+      cnameStatus: boolean,
+    ): Promise<void> => {
+      if (options?.recordResult === false) {
+        return;
+      }
+
+      await this.updateCnameStatusForStatusPageDomain({
+        domain: fullDomain,
+        cnameStatus: cnameStatus,
+      });
+    };
+
     try {
       // get the token from the domain.
 
@@ -540,10 +852,7 @@ export class Service extends DatabaseService<StatusPageDomain> {
         logger.debug(result, { fullDomain } as LogAttributes);
 
         if (result.isSuccess()) {
-          await this.updateCnameStatusForStatusPageDomain({
-            domain: fullDomain,
-            cnameStatus: true,
-          });
+          await recordResult(true);
 
           return true;
         }
@@ -573,10 +882,7 @@ export class Service extends DatabaseService<StatusPageDomain> {
         logger.debug(resultHttps, { fullDomain } as LogAttributes);
 
         if (resultHttps.isSuccess()) {
-          await this.updateCnameStatusForStatusPageDomain({
-            domain: fullDomain,
-            cnameStatus: true,
-          });
+          await recordResult(true);
 
           return true;
         }
@@ -605,10 +911,7 @@ export class Service extends DatabaseService<StatusPageDomain> {
               `No CNAME record found for ${fullDomain}. Expected record: ${StatusPageCNameRecord}`,
               { fullDomain } as LogAttributes,
             );
-            await this.updateCnameStatusForStatusPageDomain({
-              domain: fullDomain,
-              cnameStatus: false,
-            });
+            await recordResult(false);
             return false;
           }
 
@@ -622,10 +925,7 @@ export class Service extends DatabaseService<StatusPageDomain> {
               { fullDomain } as LogAttributes,
             );
 
-            await this.updateCnameStatusForStatusPageDomain({
-              domain: fullDomain,
-              cnameStatus: true,
-            });
+            await recordResult(true);
 
             return true;
           }
@@ -642,10 +942,7 @@ export class Service extends DatabaseService<StatusPageDomain> {
         logger.debug(err, { fullDomain } as LogAttributes);
       }
 
-      await this.updateCnameStatusForStatusPageDomain({
-        domain: fullDomain,
-        cnameStatus: false,
-      });
+      await recordResult(false);
 
       return false;
     } catch (err) {
@@ -654,15 +951,32 @@ export class Service extends DatabaseService<StatusPageDomain> {
       } as LogAttributes);
       logger.debug(err, { fullDomain } as LogAttributes);
 
-      await this.updateCnameStatusForStatusPageDomain({
-        domain: fullDomain,
-        cnameStatus: false,
-      });
+      await recordResult(false);
 
       return false;
     }
   }
 
+  /*
+   * Record whether this domain's certificate is being served yet.
+   *
+   * This never orders a certificate. It used to re-order whenever the HTTPS
+   * probe failed while the CNAME still checked out - and right after an
+   * order the probe fails as a rule, because nginx writes new certificates
+   * to disk every 15 minutes, so a new domain was ordered twice. The
+   * re-order could also delete a working certificate: it checks the CNAME
+   * once more, and GreenlockUtil.orderCert removes the name's certificate
+   * when that check fails, so a moment's DNS trouble between the two checks
+   * took the domain off HTTPS. The two cases that do need an order have
+   * capped sweeps of their own: checkOrderStatus re-orders a certificate
+   * that has gone missing, and the renewal run renews one that is about to
+   * expire or has expired.
+   *
+   * When the probe fails the CNAME is checked again, so a domain whose
+   * record was removed goes back to asking for it. The result is written
+   * only when it changes: this sweep visits every ordered domain every 15
+   * minutes.
+   */
   @CaptureSpan()
   public async updateSslProvisioningStatus(
     domain: StatusPageDomain,
@@ -679,6 +993,7 @@ export class Service extends DatabaseService<StatusPageDomain> {
         _id: true,
         fullDomain: true,
         cnameVerificationToken: true,
+        isSslProvisioned: true,
       },
       props: {
         isRoot: true,
@@ -690,52 +1005,25 @@ export class Service extends DatabaseService<StatusPageDomain> {
     }
 
     logger.debug(
-      `StatusPageCerts:RemoveCerts - Checking CNAME ${statusPageDomain.fullDomain}`,
+      `StatusPageCerts:CheckSslProvisioningStatus - Checking ${statusPageDomain.fullDomain}`,
       { fullDomain: statusPageDomain.fullDomain } as LogAttributes,
     );
 
-    // Check CNAME validation and if that fails. Remove certs from Greenlock.
-    const isValid: boolean = await this.isSSLProvisioned(
+    const isProvisioned: boolean = await this.isSSLProvisioned(
       statusPageDomain.fullDomain!,
       statusPageDomain.cnameVerificationToken!,
     );
 
-    if (!isValid) {
-      // check if cname is valid.
-
-      const isCnameValid: boolean = await this.isCnameValid(
-        statusPageDomain.fullDomain!,
-      );
-
-      await this.updateOneById({
-        id: statusPageDomain.id!,
-        data: {
-          isSslProvisioned: false,
-        },
-        props: {
-          isRoot: true,
-        },
-      });
+    if (!isProvisioned) {
+      // isCnameValid also records the result on the domain.
+      await this.isCnameValid(statusPageDomain.fullDomain!);
+    }
 
-      if (isCnameValid) {
-        try {
-          // if cname is valid then order cert again.
-          await this.orderCert(statusPageDomain);
-        } catch (err) {
-          logger.error(
-            "Cannot order cert for domain: " + statusPageDomain.fullDomain,
-            { fullDomain: statusPageDomain.fullDomain } as LogAttributes,
-          );
-          logger.error(err, {
-            fullDomain: statusPageDomain.fullDomain,
-          } as LogAttributes);
-        }
-      }
-    } else {
+    if (Boolean(statusPageDomain.isSslProvisioned) !== isProvisioned) {
       await this.updateOneById({
         id: statusPageDomain.id!,
         data: {
-          isSslProvisioned: true,
+          isSslProvisioned: isProvisioned,
         },
         props: {
           isRoot: true,
@@ -744,6 +1032,19 @@ export class Service extends DatabaseService<StatusPageDomain> {
     }
   }
 
+  /*
+   * Order the first certificate of every verified domain that has none yet,
+   * at most ORDER_MAX_PER_RUN per run (orderCertsForVerifiedDomainsWithoutOne).
+   *
+   * Only verified domains. This sweep used to try every domain not marked
+   * ordered, verified or not, and without a limit: for a domain whose record
+   * was not in place that was one more failed CNAME check every 15 minutes,
+   * and for one whose check failed for a moment it was a duplicate order -
+   * or, if the check failed again during the order, the deletion of its
+   * working certificate. verifyCnameWhoseCnameisNotVerified checks the
+   * unverified domains, and orders each one's certificate the moment it is
+   * verified; this sweep is the fallback for the rest.
+   */
   @CaptureSpan()
   public async orderSSLForDomainsWhichAreNotOrderedYet(): Promise<void> {
     return Telemetry.startActiveSpan<Promise<void>>({
@@ -751,34 +1052,25 @@ export class Service extends DatabaseService<StatusPageDomain> {
       options: { attributes: {} },
       fn: async (span: Span): Promise<void> => {
         try {
-          const domains: Array<StatusPageDomain> = await this.findBy({
+          const domains: Array<StatusPageDomain> = await this.findAllBy({
             query: {
+              isCnameVerified: true,
               isSslOrdered: false,
               isCustomCertificate: false, // only order for non custom certificates.
             },
             select: {
               _id: true,
               fullDomain: true,
+              isSslOrdered: true,
+              isCustomCertificate: true,
             },
-            limit: LIMIT_MAX,
             skip: 0,
             props: {
               isRoot: true,
             },
           });
 
-          for (const domain of domains) {
-            try {
-              logger.debug("Ordering SSL for domain: " + domain.fullDomain, {
-                fullDomain: domain.fullDomain,
-              } as LogAttributes);
-              await this.orderCert(domain);
-            } catch (e) {
-              logger.error(e, {
-                fullDomain: domain.fullDomain,
-              } as LogAttributes);
-            }
-          }
+          await this.orderCertsForVerifiedDomainsWithoutOne(domains);
 
           Telemetry.endSpan(span);
         } catch (err) {
@@ -793,6 +1085,14 @@ export class Service extends DatabaseService<StatusPageDomain> {
     });
   }
 
+  /*
+   * Check every domain whose CNAME is not verified yet, at most
+   * SSL_PROVISIONING_CHECK_CONCURRENCY at a time, and order the certificate
+   * of each one that is verified now straight away rather than at the next
+   * run of the order sweep. So a domain is served on its own certificate
+   * within about 15 minutes of its record going live - nginx writes new
+   * certificates every 15 minutes - whether or not anyone clicked Check now.
+   */
   @CaptureSpan()
   public async verifyCnameWhoseCnameisNotVerified(): Promise<void> {
     const domains: Array<StatusPageDomain> = await this.findBy({
@@ -802,6 +1102,8 @@ export class Service extends DatabaseService<StatusPageDomain> {
       select: {
         _id: true,
         fullDomain: true,
+        isCustomCertificate: true,
+        isSslOrdered: true,
       },
       limit: LIMIT_MAX,
       skip: 0,
@@ -810,22 +1112,99 @@ export class Service extends DatabaseService<StatusPageDomain> {
       },
     });
 
-    for (const domain of domains) {
-      try {
-        await this.isCnameValid(domain.fullDomain as string); // this will also upate the status.
-      } catch (e) {
-        logger.error(e, {
-          fullDomain: domain.fullDomain as string,
-        } as LogAttributes);
-      }
+    const verifiedNow: Array<StatusPageDomain> = [];
+
+    await ArrayUtil.forEachWithConcurrency(
+      domains,
+      Service.SSL_PROVISIONING_CHECK_CONCURRENCY,
+      async (domain: StatusPageDomain): Promise<void> => {
+        try {
+          // isCnameValid also records the result on the domain.
+          const isVerified: boolean = await this.isCnameValid(
+            domain.fullDomain as string,
+          );
+
+          if (isVerified && !domain.isCustomCertificate) {
+            verifiedNow.push(domain);
+          }
+        } catch (e) {
+          logger.error(e, {
+            fullDomain: domain.fullDomain as string,
+          } as LogAttributes);
+        }
+      },
+    );
+
+    try {
+      /*
+       * Each was verified by this run a moment ago, so the orders do not
+       * check the CNAME again.
+       */
+      await this.orderCertsForVerifiedDomainsWithoutOne(verifiedNow, {
+        cnameVerifiedJustNow: true,
+      });
+    } catch (e) {
+      // The order sweep orders them on its next run.
+      logger.error(e);
     }
   }
 
+  /*
+   * Which of these certificate domains are status page domains. The renewal
+   * run renews - and removes - only the certificates this returns, so the
+   * certificates of dashboard domains and of the primary host, which share
+   * the AcmeCertificate table, are never mistaken for status page domains
+   * whose CNAME stopped validating.
+   *
+   * A domain serving a certificate its owner uploaded still claims its
+   * Let's Encrypt certificate, which stays renewed meanwhile: nginx serves
+   * the uploaded one for that name (AcmeWriteCertificates leaves it alone),
+   * and switching back is then instant, on every nginx replica, including
+   * ones that start with an empty certificate directory.
+   */
+  @CaptureSpan()
+  public async getOwnedDomains(domains: Array<string>): Promise<Array<string>> {
+    if (domains.length === 0) {
+      return [];
+    }
+
+    const statusPageDomains: Array<StatusPageDomain> = await this.findBy({
+      query: {
+        fullDomain: QueryHelper.any(domains),
+      },
+      select: {
+        fullDomain: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    return statusPageDomains
+      .map((statusPageDomain: StatusPageDomain) => {
+        return statusPageDomain.fullDomain || "";
+      })
+      .filter((fullDomain: string) => {
+        return fullDomain.length > 0;
+      });
+  }
+
   @CaptureSpan()
   public async renewCertsWhichAreExpiringSoon(): Promise<void> {
     await GreenlockUtil.renewAllCertsWhichAreExpiringSoon({
+      getOwnedDomains: async (domains: Array<string>) => {
+        return await this.getOwnedDomains(domains);
+      },
+      /*
+       * A check that writes nothing: a DNS blip while a certificate is
+       * renewed must not mark the domain unverified and unordered while its
+       * certificate keeps serving. The provisioning sweep still notices a
+       * record that is really gone.
+       */
       validateCname: async (fullDomain: string) => {
-        return await this.isCnameValid(fullDomain);
+        return await this.isCnameValid(fullDomain, { recordResult: false });
       },
       notifyDomainRemoved: async (domain: string) => {
         // mark the domain as not ordered.
@@ -849,9 +1228,20 @@ export class Service extends DatabaseService<StatusPageDomain> {
     });
   }
 
+  /*
+   * Re-order the certificate of a domain that says it has one ordered but
+   * whose certificate is gone, at most ORDER_MAX_PER_RUN per run, picked
+   * afresh every run. It used to look the certificates up one domain at a
+   * time and re-order every missing one in the same run, without a limit.
+   *
+   * The re-order goes through orderCertIfMissing like every first order, so
+   * it never races one that Check now or the order sweep is placing. A
+   * domain whose last re-orders failed waits longer after each failure
+   * (CertificateOrderFailures).
+   */
   @CaptureSpan()
   public async checkOrderStatus(): Promise<void> {
-    const domains: Array<StatusPageDomain> = await this.findBy({
+    const domains: Array<StatusPageDomain> = await this.findAllBy({
       query: {
         isSslOrdered: true,
         isCustomCertificate: false,
@@ -859,44 +1249,63 @@ export class Service extends DatabaseService<StatusPageDomain> {
       select: {
         _id: true,
         fullDomain: true,
-        cnameVerificationToken: true,
+        isCustomCertificate: true,
       },
-      limit: LIMIT_MAX,
       skip: 0,
       props: {
         isRoot: true,
       },
     });
 
-    for (const domain of domains) {
-      if (!domain.fullDomain) {
-        continue;
-      }
+    const domainsWithName: Array<StatusPageDomain> = domains.filter(
+      (domain: StatusPageDomain) => {
+        return Boolean(domain.id) && Boolean(domain.fullDomain);
+      },
+    );
 
-      //check if cert exists in AcmeCertificate.
-      const acmeCert: AcmeCertificate | null =
-        await AcmeCertificateService.findOneBy({
-          query: {
-            domain: domain.fullDomain,
-          },
-          select: {
-            _id: true,
-          },
-          props: {
-            isRoot: true,
-          },
-        });
+    if (domainsWithName.length === 0) {
+      return;
+    }
 
-      if (!acmeCert) {
-        try {
-          // order cert again.
-          await this.orderCert(domain);
-        } catch (err) {
-          logger.error("Cannot order cert for domain: " + domain.fullDomain, {
-            fullDomain: domain.fullDomain,
-          } as LogAttributes);
-          logger.error(err, { fullDomain: domain.fullDomain } as LogAttributes);
-        }
+    const certificates: Map<string, AcmeCertificate> =
+      await GreenlockUtil.findCertificatesByDomain(
+        domainsWithName.map((domain: StatusPageDomain) => {
+          return CertificateOrder.normalizeDomain(domain.fullDomain as string);
+        }),
+      );
+
+    const now: Date = OneUptimeDate.getCurrentDate();
+
+    const domainsWithoutCertificate: Array<StatusPageDomain> =
+      await CertificateOrderFailures.withoutThoseWaitingToRetry({
+        items: domainsWithName.filter((domain: StatusPageDomain) => {
+          return !certificates.has(
+            CertificateOrder.normalizeDomain(domain.fullDomain as string),
+          );
+        }),
+        getDomain: (domain: StatusPageDomain): string => {
+          return domain.fullDomain as string;
+        },
+        now: now,
+      });
+
+    const batch: Array<StatusPageDomain> = GreenlockUtil.pickForThisRun({
+      items: domainsWithoutCertificate,
+      max: Service.ORDER_MAX_PER_RUN,
+      now: now,
+      getKey: (domain: StatusPageDomain): string => {
+        return domain.fullDomain || "";
+      },
+    });
+
+    for (const domain of batch) {
+      try {
+        await this.orderCertIfMissing(domain, { fromSweep: true });
+      } catch (err) {
+        logger.error("Cannot order cert for domain: " + domain.fullDomain, {
+          fullDomain: domain.fullDomain,
+        } as LogAttributes);
+        logger.error(err, { fullDomain: domain.fullDomain } as LogAttributes);
       }
     }
   }