@oneuptime/common 14.0.13 → 14.0.14

package/build/dist/Server/Services/StatusPageDomainService.js

@@ -8,6 +8,11 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 import GreenlockUtil from "../Utils/Greenlock/Greenlock";
+import CertificateOrder, { CertificateOrderOutcome, } from "../Utils/Greenlock/CertificateOrder";
+import CustomDomainOrders from "../Utils/Greenlock/CustomDomainOrders";
+import { CertificateOrderReason } from "../Utils/Greenlock/CertificateOrderBudget";
+import CertificateOrderFailures from "../Utils/Greenlock/CertificateOrderFailures";
+import CertificateReissueOrder from "../Utils/Greenlock/CertificateReissueOrder";
 import logger from "../Utils/Logger";
 import DatabaseService from "./DatabaseService";
 import DomainService from "./DomainService";
@@ -15,7 +20,6 @@ import LIMIT_MAX from "../../Types/Database/LimitMax";
 import BadDataException from "../../Types/Exception/BadDataException";
 import ObjectID from "../../Types/ObjectID";
 import StatusPageDomain from "../../Models/DatabaseModels/StatusPageDomain";
-import AcmeCertificateService from "./AcmeCertificateService";
 import Telemetry from "../Utils/Telemetry";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { StatusPageCNameRecord } from "../EnvironmentConfig";
@@ -23,7 +27,6 @@ import Domain from "../Types/Domain";
 import ArrayUtil from "../../Utils/Array";
 import OneUptimeDate from "../../Types/Date";
 import CertificateReissueUtil from "../../Utils/CertificateReissue";
-import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
 import QueryHelper from "../Types/Database/QueryHelper";
 const STATUS_PAGE_DOMAIN_EGRESS_LABEL = "Status page domain";
 export class Service extends DatabaseService {
@@ -117,7 +120,48 @@ export class Service extends DatabaseService {
     async removeDomainFromGreenlock(domain) {
         await GreenlockUtil.removeDomain(domain);
     }
-    async orderCert(statusPageDomain) {
+    /*
+     * The domain with its full name and whether it serves an uploaded
+     * certificate, read once when the caller did not have them both.
+     */
+    async getDomainToOrder(statusPageDomain) {
+        if (!statusPageDomain.id) {
+            throw new BadDataException("Domain ID is required to order a certificate");
+        }
+        if (statusPageDomain.fullDomain &&
+            statusPageDomain.isCustomCertificate !== undefined) {
+            return statusPageDomain;
+        }
+        const fetchedStatusPageDomain = await this.findOneBy({
+            query: {
+                _id: statusPageDomain.id.toString(),
+            },
+            select: {
+                _id: true,
+                fullDomain: true,
+                isCustomCertificate: true,
+                projectId: true,
+            },
+            props: {
+                isRoot: true,
+            },
+        });
+        if (!fetchedStatusPageDomain) {
+            throw new BadDataException("DomainModel not found");
+        }
+        if (!fetchedStatusPageDomain.fullDomain) {
+            throw new BadDataException("Unable to order certificate because domain is null");
+        }
+        return fetchedStatusPageDomain;
+    }
+    /*
+     * Order this domain's certificate and record it: a first certificate
+     * (the default), or a reissue. GreenlockUtil.orderCert places it under the
+     * name's order lock - the one handed in, or its own - and within the
+     * installation's Let's Encrypt allowance; the outcome says whether it
+     * ordered.
+     */
+    async orderCert(statusPageDomain, options) {
         var _a;
         return Telemetry.startActiveSpan({
             name: "StatusPageDomainService.orderCert",
@@ -129,46 +173,44 @@ export class Service extends DatabaseService {
             },
             fn: async (span) => {
                 try {
-                    if (!statusPageDomain.fullDomain) {
-                        const fetchedStatusPageDomain = await this.findOneBy({
-                            query: {
-                                _id: statusPageDomain.id.toString(),
+                    const domainToOrder = await this.getDomainToOrder(statusPageDomain);
+                    logger.debug("Ordering SSL for domain: " + domainToOrder.fullDomain, {
+                        fullDomain: domainToOrder.fullDomain,
+                    });
+                    const outcome = await GreenlockUtil.orderCert({
+                        domain: domainToOrder.fullDomain,
+                        reason: (options === null || options === void 0 ? void 0 : options.reason) || CertificateOrderReason.FirstCertificate,
+                        validateCname: (options === null || options === void 0 ? void 0 : options.cnameVerifiedJustNow)
+                            ? null
+                            : async (fullDomain) => {
+                                return await this.isCnameValid(fullDomain);
                             },
-                            select: {
-                                _id: true,
-                                fullDomain: true,
+                        lock: options === null || options === void 0 ? void 0 : options.lock,
+                        mayOrder: options === null || options === void 0 ? void 0 : options.mayOrder,
+                    });
+                    if (outcome === CertificateOrderOutcome.Ordered) {
+                        logger.debug("SSL ordered for domain: " + domainToOrder.fullDomain, { fullDomain: domainToOrder.fullDomain });
+                        /*
+                         * Verified as well as ordered: Let's Encrypt has just fetched
+                         * its challenge from this domain, so the domain reaches us. A
+                         * CNAME sweep that saw a DNS blip while the order ran - and
+                         * marked the domain unverified and unordered - would otherwise
+                         * leave it "ordered" but "waiting for DNS", with a certificate
+                         * the dialog had just called verified.
+                         */
+                        await this.updateOneById({
+                            id: domainToOrder.id,
+                            data: {
+                                isCnameVerified: true,
+                                isSslOrdered: true,
                             },
                             props: {
                                 isRoot: true,
                             },
                         });
-                        if (!fetchedStatusPageDomain) {
-                            throw new BadDataException("DomainModel not found");
-                        }
-                        statusPageDomain = fetchedStatusPageDomain;
-                    }
-                    if (!statusPageDomain.fullDomain) {
-                        throw new BadDataException("Unable to order certificate because domain is null");
                     }
-                    logger.debug("Ordering SSL for domain: " + statusPageDomain.fullDomain, { fullDomain: statusPageDomain.fullDomain });
-                    await GreenlockUtil.orderCert({
-                        domain: statusPageDomain.fullDomain,
-                        validateCname: async (fullDomain) => {
-                            return await this.isCnameValid(fullDomain);
-                        },
-                    });
-                    logger.debug("SSL ordered for domain: " + statusPageDomain.fullDomain, { fullDomain: statusPageDomain.fullDomain });
-                    // update the order.
-                    await this.updateOneById({
-                        id: statusPageDomain.id,
-                        data: {
-                            isSslOrdered: true,
-                        },
-                        props: {
-                            isRoot: true,
-                        },
-                    });
                     Telemetry.endSpan(span);
+                    return outcome;
                 }
                 catch (err) {
                     Telemetry.recordExceptionMarkSpanAsErrorAndEndSpan({
@@ -180,6 +222,36 @@ export class Service extends DatabaseService {
             },
         });
     }
+    /*
+     * Records these domains as ordered: they have a certificate, ordered
+     * before, so nothing is ordered. One write for all of them, and only on
+     * domains that are still verified - a CNAME check that failed meanwhile
+     * marked a domain unverified and unordered, and marking it ordered again
+     * would leave it ordered but "waiting for DNS". The next CNAME check that
+     * passes records it again.
+     */
+    async recordCertificatesAsOrdered(domainIds) {
+        if (domainIds.length === 0) {
+            return;
+        }
+        await this.updateBy({
+            query: {
+                _id: QueryHelper.any(domainIds.map((id) => {
+                    return id.toString();
+                })),
+                isCnameVerified: true,
+                isSslOrdered: false,
+            },
+            data: {
+                isSslOrdered: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+    }
     /*
      * Reissue this domain's Let's Encrypt certificate because a customer asked
      * for it, rather than because it is close to expiring.
@@ -229,50 +301,159 @@ export class Service extends DatabaseService {
             throw new BadDataException("CNAME is not verified. Please verify CNAME first before you reissue the SSL certificate.");
         }
         /*
-         * A domain that never ordered a certificate has nothing to reissue - the
-         * dashboard shows "Order Free SSL" for it, which is the correct button.
+         * A domain that never ordered a certificate has nothing to reissue. Its
+         * first certificate is ordered on its own once the CNAME is verified, and
+         * the dashboard shows Reissue SSL only after that.
          */
         if (!statusPageDomain.isSslOrdered) {
-            throw new BadDataException("No SSL certificate has been ordered for this domain yet. Please order one first.");
+            throw new BadDataException("No SSL certificate has been ordered for this domain yet. One is ordered automatically once its CNAME record is verified.");
         }
-        /*
-         * Claim the cooldown before ordering anything, and claim it by putting the
-         * cooldown condition in the WHERE clause rather than in an `if` above the
-         * write. Two clicks that arrive together both read a row that is not
-         * cooling down; only the write can decide which one of them gets to spend
-         * the CA's allowance.
-         *
-         * The stamp goes down BEFORE the order and is never rolled back on
-         * failure. A rejected order still costs a validation attempt against the
-         * shared account, and a domain whose order fails is exactly the domain
-         * somebody presses again - so a failed attempt has to start the clock the
-         * same way a successful one does.
-         */
-        const claimedRowCount = await this.updateOneBy({
-            query: {
-                _id: domainId.toString(),
-                certificateReissueRequestedAt: QueryHelper.lessThanEqualToOrNull(CertificateReissueUtil.getCooldownCutoff(now)),
+        await CertificateReissueOrder.reissue({
+            domain: statusPageDomain.fullDomain,
+            now: now,
+            lastReissueRequestedAt: statusPageDomain.certificateReissueRequestedAt,
+            claimCooldown: async () => {
+                /*
+                 * Claim the cooldown before ordering anything, and claim it by
+                 * putting the cooldown condition in the WHERE clause rather than in
+                 * an `if` above the write. Two clicks that arrive together both read
+                 * a row that is not cooling down; only the write can decide which
+                 * one of them gets to spend the CA's allowance.
+                 */
+                return await this.updateOneBy({
+                    query: {
+                        _id: domainId.toString(),
+                        certificateReissueRequestedAt: QueryHelper.lessThanEqualToOrNull(CertificateReissueUtil.getCooldownCutoff(now)),
+                    },
+                    data: {
+                        certificateReissueRequestedAt: now,
+                    },
+                    props: {
+                        isRoot: true,
+                    },
+                });
             },
-            data: {
-                certificateReissueRequestedAt: now,
+            giveCooldownBack: async () => {
+                await this.updateOneBy({
+                    query: {
+                        _id: domainId.toString(),
+                        certificateReissueRequestedAt: now,
+                    },
+                    data: {
+                        certificateReissueRequestedAt: statusPageDomain.certificateReissueRequestedAt || null,
+                    },
+                    props: {
+                        isRoot: true,
+                    },
+                });
             },
-            props: {
-                isRoot: true,
+            order: async (lock) => {
+                logger.debug("Reissuing SSL certificate for domain: " +
+                    statusPageDomain.fullDomain, { fullDomain: statusPageDomain.fullDomain });
+                return await this.orderCert(statusPageDomain, {
+                    reason: CertificateOrderReason.Reissue,
+                    lock: lock,
+                });
             },
         });
-        if (claimedRowCount === 0) {
-            if (statusPageDomain.certificateReissueRequestedAt) {
-                throw new TooManyRequestsException(CertificateReissueUtil.getCooldownMessage(statusPageDomain.certificateReissueRequestedAt, now));
-            }
-            /*
-             * The row matched the cooldown a moment ago and does not now: another
-             * request claimed it in between, or the domain was deleted. Either way
-             * the caller must not order.
-             */
-            throw new BadDataException("Could not start a certificate reissue for this domain. Please refresh the page and try again.");
+    }
+    /*
+     * Order this domain's free certificate unless it has one already, or one
+     * is being ordered for it right now (CertificateOrder.orderIfMissing): the
+     * one way a first certificate is ordered, whether by Check now, by the
+     * order sweeps or by the order-ssl API.
+     *
+     * Never for a domain that serves a certificate its owner uploaded. Throws
+     * what the order throws, so a caller can show why.
+     */
+    async orderCertIfMissing(statusPageDomain, options) {
+        const domainToOrder = await this.getDomainToOrder(statusPageDomain);
+        // The sweeps never order for one either: it serves the upload.
+        if (domainToOrder.isCustomCertificate) {
+            throw new BadDataException("This domain uses a certificate you uploaded, so there is no free SSL certificate to order for it.");
         }
-        logger.debug("Reissuing SSL certificate for domain: " + statusPageDomain.fullDomain, { fullDomain: statusPageDomain.fullDomain });
-        await this.orderCert(statusPageDomain);
+        const domainId = domainToOrder.id;
+        return await CertificateOrder.orderIfMissing({
+            domain: domainToOrder.fullDomain,
+            renewIfExpired: options === null || options === void 0 ? void 0 : options.renewIfExpired,
+            recordAsOrdered: async () => {
+                await this.recordCertificatesAsOrdered([domainId]);
+            },
+            order: async (lock) => {
+                var _a;
+                return await this.orderCert(domainToOrder, {
+                    reason: CertificateOrderReason.FirstCertificate,
+                    lock: lock,
+                    cnameVerifiedJustNow: options === null || options === void 0 ? void 0 : options.cnameVerifiedJustNow,
+                    mayOrder: CertificateOrder.getOrderShare({
+                        sweep: (options === null || options === void 0 ? void 0 : options.fromSweep)
+                            ? {
+                                budget: Service.SWEEP_ORDER_BUDGET,
+                                maxPerWindow: Service.ORDER_MAX_PER_RUN,
+                            }
+                            : undefined,
+                        onDemand: (options === null || options === void 0 ? void 0 : options.onDemand)
+                            ? { projectId: (_a = domainToOrder.projectId) === null || _a === void 0 ? void 0 : _a.toString() }
+                            : undefined,
+                    }),
+                });
+            },
+        });
+    }
+    /*
+     * Called by verify-cname once it has found the domain's CNAME record:
+     * order the domain's free certificate now rather than at the next run of
+     * the order sweep, and say what happens to its certificate next
+     * (CustomDomainOrders.orderOnceCnameIsVerified, which dashboard domains
+     * share). Never rejects.
+     */
+    async orderCertOnceCnameIsVerified(statusPageDomain, options) {
+        return await CustomDomainOrders.orderOnceCnameIsVerified({
+            domain: statusPageDomain,
+            waitInMs: options === null || options === void 0 ? void 0 : options.waitInMs,
+            recordAsOrdered: async () => {
+                await this.recordCertificatesAsOrdered([statusPageDomain.id]);
+            },
+            orderIfMissing: async () => {
+                return await this.orderCertIfMissing(statusPageDomain, {
+                    onDemand: true,
+                    // The route found the record a moment ago.
+                    cnameVerifiedJustNow: true,
+                    renewIfExpired: true,
+                });
+            },
+        });
+    }
+    /*
+     * Order the first certificate of each of these domains, whose CNAME is
+     * verified: at most ORDER_MAX_PER_RUN orders, picked afresh every run; a
+     * domain that has a certificate already is only recorded as ordered, and
+     * one whose orders keep failing waits longer after each failure
+     * (CustomDomainOrders.orderForVerifiedDomainsWithoutOne, which the
+     * dashboard sweeps share).
+     */
+    async orderCertsForVerifiedDomainsWithoutOne(domains, options) {
+        await CustomDomainOrders.orderForVerifiedDomainsWithoutOne({
+            domains: domains,
+            maxPerRun: Service.ORDER_MAX_PER_RUN,
+            recordAsOrdered: async (domainIds) => {
+                await this.recordCertificatesAsOrdered(domainIds);
+            },
+            orderIfMissing: async (domain) => {
+                return await this.orderCertIfMissing(domain, {
+                    fromSweep: true,
+                    cnameVerifiedJustNow: options === null || options === void 0 ? void 0 : options.cnameVerifiedJustNow,
+                });
+            },
+        });
+    }
+    /*
+     * Where the certificates of these domains stand, for the Custom Domains
+     * page's Status column (CustomDomainCertificates): each one's expiry, and
+     * its last failed order while no order since has succeeded.
+     */
+    async getCertificates(domains) {
+        return await CustomDomainOrders.getCertificates(domains);
     }
     async updateSslProvisioningStatusForAllDomains() {
         const domains = await this.findBy({
@@ -358,7 +539,16 @@ export class Service extends DatabaseService {
             });
         }
     }
-    async isCnameValid(fullDomain) {
+    async isCnameValid(fullDomain, options) {
+        const recordResult = async (cnameStatus) => {
+            if ((options === null || options === void 0 ? void 0 : options.recordResult) === false) {
+                return;
+            }
+            await this.updateCnameStatusForStatusPageDomain({
+                domain: fullDomain,
+                cnameStatus: cnameStatus,
+            });
+        };
         try {
             // get the token from the domain.
             logger.debug("Checking for CNAME " + fullDomain, {
@@ -394,10 +584,7 @@ export class Service extends DatabaseService {
                 });
                 logger.debug(result, { fullDomain });
                 if (result.isSuccess()) {
-                    await this.updateCnameStatusForStatusPageDomain({
-                        domain: fullDomain,
-                        cnameStatus: true,
-                    });
+                    await recordResult(true);
                     return true;
                 }
             }
@@ -421,10 +608,7 @@ export class Service extends DatabaseService {
                 });
                 logger.debug(resultHttps, { fullDomain });
                 if (resultHttps.isSuccess()) {
-                    await this.updateCnameStatusForStatusPageDomain({
-                        domain: fullDomain,
-                        cnameStatus: true,
-                    });
+                    await recordResult(true);
                     return true;
                 }
             }
@@ -446,20 +630,14 @@ export class Service extends DatabaseService {
                     }
                     if (!cnameRecord) {
                         logger.debug(`No CNAME record found for ${fullDomain}. Expected record: ${StatusPageCNameRecord}`, { fullDomain });
-                        await this.updateCnameStatusForStatusPageDomain({
-                            domain: fullDomain,
-                            cnameStatus: false,
-                        });
+                        await recordResult(false);
                         return false;
                     }
                     if (cnameRecord &&
                         cnameRecord.trim().toLocaleLowerCase() ===
                             StatusPageCNameRecord.trim().toLocaleLowerCase()) {
                         logger.debug(`CNAME record for ${fullDomain} matches the expected record: ${StatusPageCNameRecord}`, { fullDomain });
-                        await this.updateCnameStatusForStatusPageDomain({
-                            domain: fullDomain,
-                            cnameStatus: true,
-                        });
+                        await recordResult(true);
                         return true;
                     }
                     logger.debug(`CNAME record for ${fullDomain} is ${cnameRecord} and it does not match the expected record: ${StatusPageCNameRecord}`, { fullDomain });
@@ -471,10 +649,7 @@ export class Service extends DatabaseService {
                 });
                 logger.debug(err, { fullDomain });
             }
-            await this.updateCnameStatusForStatusPageDomain({
-                domain: fullDomain,
-                cnameStatus: false,
-            });
+            await recordResult(false);
             return false;
         }
         catch (err) {
@@ -482,13 +657,30 @@ export class Service extends DatabaseService {
                 fullDomain,
             });
             logger.debug(err, { fullDomain });
-            await this.updateCnameStatusForStatusPageDomain({
-                domain: fullDomain,
-                cnameStatus: false,
-            });
+            await recordResult(false);
             return false;
         }
     }
+    /*
+     * Record whether this domain's certificate is being served yet.
+     *
+     * This never orders a certificate. It used to re-order whenever the HTTPS
+     * probe failed while the CNAME still checked out - and right after an
+     * order the probe fails as a rule, because nginx writes new certificates
+     * to disk every 15 minutes, so a new domain was ordered twice. The
+     * re-order could also delete a working certificate: it checks the CNAME
+     * once more, and GreenlockUtil.orderCert removes the name's certificate
+     * when that check fails, so a moment's DNS trouble between the two checks
+     * took the domain off HTTPS. The two cases that do need an order have
+     * capped sweeps of their own: checkOrderStatus re-orders a certificate
+     * that has gone missing, and the renewal run renews one that is about to
+     * expire or has expired.
+     *
+     * When the probe fails the CNAME is checked again, so a domain whose
+     * record was removed goes back to asking for it. The result is written
+     * only when it changes: this sweep visits every ordered domain every 15
+     * minutes.
+     */
     async updateSslProvisioningStatus(domain) {
         var _a;
         if (!domain.id) {
@@ -502,6 +694,7 @@ export class Service extends DatabaseService {
                 _id: true,
                 fullDomain: true,
                 cnameVerificationToken: true,
+                isSslProvisioned: true,
             },
             props: {
                 isRoot: true,
@@ -510,39 +703,17 @@ export class Service extends DatabaseService {
         if (!statusPageDomain) {
             throw new BadDataException("DomainModel not found");
         }
-        logger.debug(`StatusPageCerts:RemoveCerts - Checking CNAME ${statusPageDomain.fullDomain}`, { fullDomain: statusPageDomain.fullDomain });
-        // Check CNAME validation and if that fails. Remove certs from Greenlock.
-        const isValid = await this.isSSLProvisioned(statusPageDomain.fullDomain, statusPageDomain.cnameVerificationToken);
-        if (!isValid) {
-            // check if cname is valid.
-            const isCnameValid = await this.isCnameValid(statusPageDomain.fullDomain);
-            await this.updateOneById({
-                id: statusPageDomain.id,
-                data: {
-                    isSslProvisioned: false,
-                },
-                props: {
-                    isRoot: true,
-                },
-            });
-            if (isCnameValid) {
-                try {
-                    // if cname is valid then order cert again.
-                    await this.orderCert(statusPageDomain);
-                }
-                catch (err) {
-                    logger.error("Cannot order cert for domain: " + statusPageDomain.fullDomain, { fullDomain: statusPageDomain.fullDomain });
-                    logger.error(err, {
-                        fullDomain: statusPageDomain.fullDomain,
-                    });
-                }
-            }
+        logger.debug(`StatusPageCerts:CheckSslProvisioningStatus - Checking ${statusPageDomain.fullDomain}`, { fullDomain: statusPageDomain.fullDomain });
+        const isProvisioned = await this.isSSLProvisioned(statusPageDomain.fullDomain, statusPageDomain.cnameVerificationToken);
+        if (!isProvisioned) {
+            // isCnameValid also records the result on the domain.
+            await this.isCnameValid(statusPageDomain.fullDomain);
         }
-        else {
+        if (Boolean(statusPageDomain.isSslProvisioned) !== isProvisioned) {
             await this.updateOneById({
                 id: statusPageDomain.id,
                 data: {
-                    isSslProvisioned: true,
+                    isSslProvisioned: isProvisioned,
                 },
                 props: {
                     isRoot: true,
@@ -550,40 +721,43 @@ export class Service extends DatabaseService {
             });
         }
     }
+    /*
+     * Order the first certificate of every verified domain that has none yet,
+     * at most ORDER_MAX_PER_RUN per run (orderCertsForVerifiedDomainsWithoutOne).
+     *
+     * Only verified domains. This sweep used to try every domain not marked
+     * ordered, verified or not, and without a limit: for a domain whose record
+     * was not in place that was one more failed CNAME check every 15 minutes,
+     * and for one whose check failed for a moment it was a duplicate order -
+     * or, if the check failed again during the order, the deletion of its
+     * working certificate. verifyCnameWhoseCnameisNotVerified checks the
+     * unverified domains, and orders each one's certificate the moment it is
+     * verified; this sweep is the fallback for the rest.
+     */
     async orderSSLForDomainsWhichAreNotOrderedYet() {
         return Telemetry.startActiveSpan({
             name: "StatusPageDomainService.orderSSLForDomainsWhichAreNotOrderedYet",
             options: { attributes: {} },
             fn: async (span) => {
                 try {
-                    const domains = await this.findBy({
+                    const domains = await this.findAllBy({
                         query: {
+                            isCnameVerified: true,
                             isSslOrdered: false,
                             isCustomCertificate: false, // only order for non custom certificates.
                         },
                         select: {
                             _id: true,
                             fullDomain: true,
+                            isSslOrdered: true,
+                            isCustomCertificate: true,
                         },
-                        limit: LIMIT_MAX,
                         skip: 0,
                         props: {
                             isRoot: true,
                         },
                     });
-                    for (const domain of domains) {
-                        try {
-                            logger.debug("Ordering SSL for domain: " + domain.fullDomain, {
-                                fullDomain: domain.fullDomain,
-                            });
-                            await this.orderCert(domain);
-                        }
-                        catch (e) {
-                            logger.error(e, {
-                                fullDomain: domain.fullDomain,
-                            });
-                        }
-                    }
+                    await this.orderCertsForVerifiedDomainsWithoutOne(domains);
                     Telemetry.endSpan(span);
                 }
                 catch (err) {
@@ -596,6 +770,14 @@ export class Service extends DatabaseService {
             },
         });
     }
+    /*
+     * Check every domain whose CNAME is not verified yet, at most
+     * SSL_PROVISIONING_CHECK_CONCURRENCY at a time, and order the certificate
+     * of each one that is verified now straight away rather than at the next
+     * run of the order sweep. So a domain is served on its own certificate
+     * within about 15 minutes of its record going live - nginx writes new
+     * certificates every 15 minutes - whether or not anyone clicked Check now.
+     */
     async verifyCnameWhoseCnameisNotVerified() {
         const domains = await this.findBy({
             query: {
@@ -604,6 +786,8 @@ export class Service extends DatabaseService {
             select: {
                 _id: true,
                 fullDomain: true,
+                isCustomCertificate: true,
+                isSslOrdered: true,
             },
             limit: LIMIT_MAX,
             skip: 0,
@@ -611,21 +795,86 @@ export class Service extends DatabaseService {
                 isRoot: true,
             },
         });
-        for (const domain of domains) {
+        const verifiedNow = [];
+        await ArrayUtil.forEachWithConcurrency(domains, Service.SSL_PROVISIONING_CHECK_CONCURRENCY, async (domain) => {
             try {
-                await this.isCnameValid(domain.fullDomain); // this will also upate the status.
+                // isCnameValid also records the result on the domain.
+                const isVerified = await this.isCnameValid(domain.fullDomain);
+                if (isVerified && !domain.isCustomCertificate) {
+                    verifiedNow.push(domain);
+                }
             }
             catch (e) {
                 logger.error(e, {
                     fullDomain: domain.fullDomain,
                 });
             }
+        });
+        try {
+            /*
+             * Each was verified by this run a moment ago, so the orders do not
+             * check the CNAME again.
+             */
+            await this.orderCertsForVerifiedDomainsWithoutOne(verifiedNow, {
+                cnameVerifiedJustNow: true,
+            });
+        }
+        catch (e) {
+            // The order sweep orders them on its next run.
+            logger.error(e);
         }
     }
+    /*
+     * Which of these certificate domains are status page domains. The renewal
+     * run renews - and removes - only the certificates this returns, so the
+     * certificates of dashboard domains and of the primary host, which share
+     * the AcmeCertificate table, are never mistaken for status page domains
+     * whose CNAME stopped validating.
+     *
+     * A domain serving a certificate its owner uploaded still claims its
+     * Let's Encrypt certificate, which stays renewed meanwhile: nginx serves
+     * the uploaded one for that name (AcmeWriteCertificates leaves it alone),
+     * and switching back is then instant, on every nginx replica, including
+     * ones that start with an empty certificate directory.
+     */
+    async getOwnedDomains(domains) {
+        if (domains.length === 0) {
+            return [];
+        }
+        const statusPageDomains = await this.findBy({
+            query: {
+                fullDomain: QueryHelper.any(domains),
+            },
+            select: {
+                fullDomain: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        return statusPageDomains
+            .map((statusPageDomain) => {
+            return statusPageDomain.fullDomain || "";
+        })
+            .filter((fullDomain) => {
+            return fullDomain.length > 0;
+        });
+    }
     async renewCertsWhichAreExpiringSoon() {
         await GreenlockUtil.renewAllCertsWhichAreExpiringSoon({
+            getOwnedDomains: async (domains) => {
+                return await this.getOwnedDomains(domains);
+            },
+            /*
+             * A check that writes nothing: a DNS blip while a certificate is
+             * renewed must not mark the domain unverified and unordered while its
+             * certificate keeps serving. The provisioning sweep still notices a
+             * record that is really gone.
+             */
             validateCname: async (fullDomain) => {
-                return await this.isCnameValid(fullDomain);
+                return await this.isCnameValid(fullDomain, { recordResult: false });
             },
             notifyDomainRemoved: async (domain) => {
                 // mark the domain as not ordered.
@@ -647,8 +896,19 @@ export class Service extends DatabaseService {
             },
         });
     }
+    /*
+     * Re-order the certificate of a domain that says it has one ordered but
+     * whose certificate is gone, at most ORDER_MAX_PER_RUN per run, picked
+     * afresh every run. It used to look the certificates up one domain at a
+     * time and re-order every missing one in the same run, without a limit.
+     *
+     * The re-order goes through orderCertIfMissing like every first order, so
+     * it never races one that Check now or the order sweep is placing. A
+     * domain whose last re-orders failed waits longer after each failure
+     * (CertificateOrderFailures).
+     */
     async checkOrderStatus() {
-        const domains = await this.findBy({
+        const domains = await this.findAllBy({
             query: {
                 isSslOrdered: true,
                 isCustomCertificate: false,
@@ -656,41 +916,49 @@ export class Service extends DatabaseService {
             select: {
                 _id: true,
                 fullDomain: true,
-                cnameVerificationToken: true,
+                isCustomCertificate: true,
             },
-            limit: LIMIT_MAX,
             skip: 0,
             props: {
                 isRoot: true,
             },
         });
-        for (const domain of domains) {
-            if (!domain.fullDomain) {
-                continue;
+        const domainsWithName = domains.filter((domain) => {
+            return Boolean(domain.id) && Boolean(domain.fullDomain);
+        });
+        if (domainsWithName.length === 0) {
+            return;
+        }
+        const certificates = await GreenlockUtil.findCertificatesByDomain(domainsWithName.map((domain) => {
+            return CertificateOrder.normalizeDomain(domain.fullDomain);
+        }));
+        const now = OneUptimeDate.getCurrentDate();
+        const domainsWithoutCertificate = await CertificateOrderFailures.withoutThoseWaitingToRetry({
+            items: domainsWithName.filter((domain) => {
+                return !certificates.has(CertificateOrder.normalizeDomain(domain.fullDomain));
+            }),
+            getDomain: (domain) => {
+                return domain.fullDomain;
+            },
+            now: now,
+        });
+        const batch = GreenlockUtil.pickForThisRun({
+            items: domainsWithoutCertificate,
+            max: Service.ORDER_MAX_PER_RUN,
+            now: now,
+            getKey: (domain) => {
+                return domain.fullDomain || "";
+            },
+        });
+        for (const domain of batch) {
+            try {
+                await this.orderCertIfMissing(domain, { fromSweep: true });
             }
-            //check if cert exists in AcmeCertificate.
-            const acmeCert = await AcmeCertificateService.findOneBy({
-                query: {
-                    domain: domain.fullDomain,
-                },
-                select: {
-                    _id: true,
-                },
-                props: {
-                    isRoot: true,
-                },
-            });
-            if (!acmeCert) {
-                try {
-                    // order cert again.
-                    await this.orderCert(domain);
-                }
-                catch (err) {
-                    logger.error("Cannot order cert for domain: " + domain.fullDomain, {
-                        fullDomain: domain.fullDomain,
-                    });
-                    logger.error(err, { fullDomain: domain.fullDomain });
-                }
+            catch (err) {
+                logger.error("Cannot order cert for domain: " + domain.fullDomain, {
+                    fullDomain: domain.fullDomain,
+                });
+                logger.error(err, { fullDomain: domain.fullDomain });
             }
         }
     }
@@ -703,6 +971,35 @@ export class Service extends DatabaseService {
  * shared infrastructure sitting in front of those domains.
  */
 Service.SSL_PROVISIONING_CHECK_CONCURRENCY = 10;
+/*
+ * How many certificates the sweeps that order them may order in one
+ * 15-minute window, between them: the order for the domains the CNAME
+ * sweep has just verified, the order sweep for every verified domain still
+ * without a certificate, and the re-order of a certificate that has gone
+ * missing. They draw from one budget (CertificateOrder.takeOrderSlot,
+ * SWEEP_ORDER_BUDGET), and each picks its batch afresh every run
+ * (GreenlockUtil.pickForThisRun), so a domain whose order keeps failing -
+ * a CAA record that leaves Let's Encrypt out, a name it has paused -
+ * cannot hold a slot the others are waiting for, and waits longer after
+ * each failed order (CertificateOrderFailures). Check now is not counted
+ * here: it has a window of its own per domain
+ * (CertificateOrder.claimOnDemandOrder) and the on-demand budget.
+ *
+ * Every order spends from the one Let's Encrypt account the whole
+ * installation shares, 300 new orders per three hours, renewals included.
+ * Whatever these caps allow, no order is placed beyond the installation's
+ * own budget for that account (CertificateOrderBudget), which every order
+ * of every kind draws from. In a steady state these sweeps order a few
+ * certificates a day: a domain whose owner clicks Check now is ordered by
+ * that request. The cap is for a backlog, such as the domains the order
+ * sweep used to retry every 15 minutes without any limit, which is now
+ * worked off a few domains a run the way the dashboard sweeps
+ * (DashboardDomainService.ORDER_MAX_PER_RUN) and renewals
+ * (GreenlockUtil.RENEW_MAX_PER_RUN) are.
+ */
+Service.ORDER_MAX_PER_RUN = 5;
+// The budget the ordering sweeps share (CertificateOrder.takeOrderSlot).
+Service.SWEEP_ORDER_BUDGET = "StatusPageDomainSweeps";
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
@@ -736,7 +1033,7 @@ __decorate([
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [StatusPageDomain]),
+    __metadata("design:paramtypes", [StatusPageDomain, Object]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "orderCert", null);
 __decorate([
@@ -745,6 +1042,24 @@ __decorate([
     __metadata("design:paramtypes", [ObjectID]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "reissueCert", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [StatusPageDomain, Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "orderCertIfMissing", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [StatusPageDomain, Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "orderCertOnceCnameIsVerified", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getCertificates", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
@@ -760,7 +1075,7 @@ __decorate([
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String]),
+    __metadata("design:paramtypes", [String, Object]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "isCnameValid", null);
 __decorate([
@@ -781,6 +1096,12 @@ __decorate([
     __metadata("design:paramtypes", []),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "verifyCnameWhoseCnameisNotVerified", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getOwnedDomains", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),