@oneuptime/common 14.0.13 → 14.0.14

package/build/dist/Server/Services/DashboardDomainService.js

@@ -8,6 +8,11 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 import GreenlockUtil from "../Utils/Greenlock/Greenlock";
+import CertificateOrder, { CertificateOrderOutcome, } from "../Utils/Greenlock/CertificateOrder";
+import CustomDomainOrders from "../Utils/Greenlock/CustomDomainOrders";
+import { CertificateOrderReason } from "../Utils/Greenlock/CertificateOrderBudget";
+import CertificateOrderFailures from "../Utils/Greenlock/CertificateOrderFailures";
+import CertificateReissueOrder from "../Utils/Greenlock/CertificateReissueOrder";
 import logger from "../Utils/Logger";
 import DatabaseService from "./DatabaseService";
 import DomainService from "./DomainService";
@@ -15,15 +20,14 @@ import LIMIT_MAX from "../../Types/Database/LimitMax";
 import BadDataException from "../../Types/Exception/BadDataException";
 import ObjectID from "../../Types/ObjectID";
 import DashboardDomain from "../../Models/DatabaseModels/DashboardDomain";
-import AcmeCertificateService from "./AcmeCertificateService";
 import Telemetry from "../Utils/Telemetry";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { DashboardCNameRecord } from "../EnvironmentConfig";
 import Domain from "../Types/Domain";
 import OneUptimeDate from "../../Types/Date";
 import CertificateReissueUtil from "../../Utils/CertificateReissue";
-import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
 import QueryHelper from "../Types/Database/QueryHelper";
+import ArrayUtil from "../../Utils/Array";
 const DASHBOARD_DOMAIN_EGRESS_LABEL = "Dashboard domain";
 export class Service extends DatabaseService {
     constructor() {
@@ -116,7 +120,49 @@ export class Service extends DatabaseService {
     async removeDomainFromGreenlock(domain) {
         await GreenlockUtil.removeDomain(domain);
     }
-    async orderCert(dashboardDomain) {
+    /*
+     * The domain with its full name and whether it serves an uploaded
+     * certificate, read once when the caller did not have them both.
+     */
+    async getDomainToOrder(dashboardDomain) {
+        if (!dashboardDomain.id) {
+            throw new BadDataException("Domain ID is required to order a certificate");
+        }
+        if (dashboardDomain.fullDomain &&
+            dashboardDomain.isCustomCertificate !== undefined) {
+            return dashboardDomain;
+        }
+        const fetchedDashboardDomain = await this.findOneBy({
+            query: {
+                _id: dashboardDomain.id.toString(),
+            },
+            select: {
+                _id: true,
+                fullDomain: true,
+                isCustomCertificate: true,
+                projectId: true,
+            },
+            props: {
+                isRoot: true,
+            },
+        });
+        if (!fetchedDashboardDomain) {
+            throw new BadDataException("DomainModel not found");
+        }
+        if (!fetchedDashboardDomain.fullDomain) {
+            throw new BadDataException("Unable to order certificate because domain is null");
+        }
+        return fetchedDashboardDomain;
+    }
+    /*
+     * Order this domain's certificate and record it: a first certificate
+     * (the default), or a reissue. GreenlockUtil.orderCert places it under the
+     * name's order lock - the one handed in, or its own - and within the
+     * installation's Let's Encrypt allowance; the outcome says whether it
+     * ordered. A CNAME check that fails refuses the order and no longer
+     * deletes the domain's certificate.
+     */
+    async orderCert(dashboardDomain, options) {
         var _a;
         return Telemetry.startActiveSpan({
             name: "DashboardDomainService.orderCert",
@@ -128,45 +174,40 @@ export class Service extends DatabaseService {
             },
             fn: async (span) => {
                 try {
-                    if (!dashboardDomain.fullDomain) {
-                        const fetchedDashboardDomain = await this.findOneBy({
-                            query: {
-                                _id: dashboardDomain.id.toString(),
+                    const domainToOrder = await this.getDomainToOrder(dashboardDomain);
+                    logger.debug("Ordering SSL for domain: " + domainToOrder.fullDomain, {
+                        fullDomain: domainToOrder.fullDomain,
+                    });
+                    const outcome = await GreenlockUtil.orderCert({
+                        domain: domainToOrder.fullDomain,
+                        reason: (options === null || options === void 0 ? void 0 : options.reason) || CertificateOrderReason.FirstCertificate,
+                        validateCname: (options === null || options === void 0 ? void 0 : options.cnameVerifiedJustNow)
+                            ? null
+                            : async (fullDomain) => {
+                                return await this.isCnameValid(fullDomain);
                             },
-                            select: {
-                                _id: true,
-                                fullDomain: true,
+                        lock: options === null || options === void 0 ? void 0 : options.lock,
+                        mayOrder: options === null || options === void 0 ? void 0 : options.mayOrder,
+                    });
+                    if (outcome === CertificateOrderOutcome.Ordered) {
+                        logger.debug("SSL ordered for domain: " + domainToOrder.fullDomain, { fullDomain: domainToOrder.fullDomain });
+                        /*
+                         * Verified as well as ordered: Let's Encrypt has just fetched
+                         * its challenge from this domain, so the domain reaches us.
+                         */
+                        await this.updateOneById({
+                            id: domainToOrder.id,
+                            data: {
+                                isCnameVerified: true,
+                                isSslOrdered: true,
                             },
                             props: {
                                 isRoot: true,
                             },
                         });
-                        if (!fetchedDashboardDomain) {
-                            throw new BadDataException("DomainModel not found");
-                        }
-                        dashboardDomain = fetchedDashboardDomain;
-                    }
-                    if (!dashboardDomain.fullDomain) {
-                        throw new BadDataException("Unable to order certificate because domain is null");
                     }
-                    logger.debug("Ordering SSL for domain: " + dashboardDomain.fullDomain, { fullDomain: dashboardDomain.fullDomain });
-                    await GreenlockUtil.orderCert({
-                        domain: dashboardDomain.fullDomain,
-                        validateCname: async (fullDomain) => {
-                            return await this.isCnameValid(fullDomain);
-                        },
-                    });
-                    logger.debug("SSL ordered for domain: " + dashboardDomain.fullDomain, { fullDomain: dashboardDomain.fullDomain });
-                    await this.updateOneById({
-                        id: dashboardDomain.id,
-                        data: {
-                            isSslOrdered: true,
-                        },
-                        props: {
-                            isRoot: true,
-                        },
-                    });
                     Telemetry.endSpan(span);
+                    return outcome;
                 }
                 catch (err) {
                     Telemetry.recordExceptionMarkSpanAsErrorAndEndSpan({
@@ -178,6 +219,110 @@ export class Service extends DatabaseService {
             },
         });
     }
+    /*
+     * Records these domains as ordered: they have a certificate, ordered
+     * before, so nothing is ordered. One write for all of them, and only on
+     * domains that are still verified (see StatusPageDomainService's twin).
+     */
+    async recordCertificatesAsOrdered(domainIds) {
+        if (domainIds.length === 0) {
+            return;
+        }
+        await this.updateBy({
+            query: {
+                _id: QueryHelper.any(domainIds.map((id) => {
+                    return id.toString();
+                })),
+                isCnameVerified: true,
+                isSslOrdered: false,
+            },
+            data: {
+                isSslOrdered: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+    }
+    /*
+     * Order this domain's free certificate unless it has one already, or one
+     * is being ordered for it right now (CertificateOrder.orderIfMissing): the
+     * one way a first certificate is ordered, whether by Check now, by the
+     * dashboard sweeps or by the order-ssl API, so a name is never ordered
+     * twice. Never for a domain that serves a certificate its owner uploaded.
+     * Throws what the order throws, so a caller can show why.
+     */
+    async orderCertIfMissing(dashboardDomain, options) {
+        const domainToOrder = await this.getDomainToOrder(dashboardDomain);
+        if (domainToOrder.isCustomCertificate) {
+            throw new BadDataException("This domain uses a certificate you uploaded, so there is no free SSL certificate to order for it.");
+        }
+        const domainId = domainToOrder.id;
+        return await CertificateOrder.orderIfMissing({
+            domain: domainToOrder.fullDomain,
+            renewIfExpired: options === null || options === void 0 ? void 0 : options.renewIfExpired,
+            recordAsOrdered: async () => {
+                await this.recordCertificatesAsOrdered([domainId]);
+            },
+            order: async (lock) => {
+                var _a;
+                return await this.orderCert(domainToOrder, {
+                    reason: CertificateOrderReason.FirstCertificate,
+                    lock: lock,
+                    cnameVerifiedJustNow: options === null || options === void 0 ? void 0 : options.cnameVerifiedJustNow,
+                    mayOrder: CertificateOrder.getOrderShare({
+                        sweep: (options === null || options === void 0 ? void 0 : options.fromSweep)
+                            ? {
+                                budget: Service.SWEEP_ORDER_BUDGET,
+                                maxPerWindow: Service.ORDER_MAX_PER_RUN,
+                            }
+                            : undefined,
+                        onDemand: (options === null || options === void 0 ? void 0 : options.onDemand)
+                            ? { projectId: (_a = domainToOrder.projectId) === null || _a === void 0 ? void 0 : _a.toString() }
+                            : undefined,
+                    }),
+                });
+            },
+        });
+    }
+    /*
+     * Called by verify-cname - the domain's Check now - once it has found the
+     * domain's CNAME record: order the domain's free certificate now rather
+     * than at the next run of the order sweep, and say what happens to its
+     * certificate next. The same step as a status page domain's
+     * (CustomDomainOrders.orderOnceCnameIsVerified): one on-demand order per
+     * domain per 15 minutes, counted against the domain's project, under the
+     * name's lock and within the installation's Let's Encrypt allowance; never
+     * for an uploaded certificate; Issued only for a certificate that is there
+     * and has not expired. Never rejects.
+     */
+    async orderCertOnceCnameIsVerified(dashboardDomain, options) {
+        return await CustomDomainOrders.orderOnceCnameIsVerified({
+            domain: dashboardDomain,
+            waitInMs: options === null || options === void 0 ? void 0 : options.waitInMs,
+            recordAsOrdered: async () => {
+                await this.recordCertificatesAsOrdered([dashboardDomain.id]);
+            },
+            orderIfMissing: async () => {
+                return await this.orderCertIfMissing(dashboardDomain, {
+                    onDemand: true,
+                    // The route found the record a moment ago.
+                    cnameVerifiedJustNow: true,
+                    renewIfExpired: true,
+                });
+            },
+        });
+    }
+    /*
+     * Where the certificates of these domains stand, for the Custom Domains
+     * page's Status column (CustomDomainCertificates): each one's expiry, and
+     * its last failed order while no order since has succeeded.
+     */
+    async getCertificates(domains) {
+        return await CustomDomainOrders.getCertificates(domains);
+    }
     /*
      * Reissue this domain's Let's Encrypt certificate because a customer asked
      * for it, rather than because it is close to expiring.
@@ -227,50 +372,60 @@ export class Service extends DatabaseService {
             throw new BadDataException("CNAME is not verified. Please verify CNAME first before you reissue the SSL certificate.");
         }
         /*
-         * A domain that never ordered a certificate has nothing to reissue - the
-         * dashboard shows "Order Free SSL" for it, which is the correct button.
+         * A domain that never ordered a certificate has nothing to reissue. Its
+         * first certificate is ordered on its own once the CNAME is verified, and
+         * the dashboard shows Reissue SSL only after that.
          */
         if (!dashboardDomain.isSslOrdered) {
-            throw new BadDataException("No SSL certificate has been ordered for this domain yet. Please order one first.");
+            throw new BadDataException("No SSL certificate has been ordered for this domain yet. One is ordered automatically once its CNAME record is verified.");
         }
-        /*
-         * Claim the cooldown before ordering anything, and claim it by putting the
-         * cooldown condition in the WHERE clause rather than in an `if` above the
-         * write. Two clicks that arrive together both read a row that is not
-         * cooling down; only the write can decide which one of them gets to spend
-         * the CA's allowance.
-         *
-         * The stamp goes down BEFORE the order and is never rolled back on
-         * failure. A rejected order still costs a validation attempt against the
-         * shared account, and a domain whose order fails is exactly the domain
-         * somebody presses again - so a failed attempt has to start the clock the
-         * same way a successful one does.
-         */
-        const claimedRowCount = await this.updateOneBy({
-            query: {
-                _id: domainId.toString(),
-                certificateReissueRequestedAt: QueryHelper.lessThanEqualToOrNull(CertificateReissueUtil.getCooldownCutoff(now)),
+        await CertificateReissueOrder.reissue({
+            domain: dashboardDomain.fullDomain,
+            now: now,
+            lastReissueRequestedAt: dashboardDomain.certificateReissueRequestedAt,
+            claimCooldown: async () => {
+                /*
+                 * Claim the cooldown before ordering anything, and claim it by
+                 * putting the cooldown condition in the WHERE clause rather than in
+                 * an `if` above the write. Two clicks that arrive together both read
+                 * a row that is not cooling down; only the write can decide which
+                 * one of them gets to spend the CA's allowance.
+                 */
+                return await this.updateOneBy({
+                    query: {
+                        _id: domainId.toString(),
+                        certificateReissueRequestedAt: QueryHelper.lessThanEqualToOrNull(CertificateReissueUtil.getCooldownCutoff(now)),
+                    },
+                    data: {
+                        certificateReissueRequestedAt: now,
+                    },
+                    props: {
+                        isRoot: true,
+                    },
+                });
             },
-            data: {
-                certificateReissueRequestedAt: now,
+            giveCooldownBack: async () => {
+                await this.updateOneBy({
+                    query: {
+                        _id: domainId.toString(),
+                        certificateReissueRequestedAt: now,
+                    },
+                    data: {
+                        certificateReissueRequestedAt: dashboardDomain.certificateReissueRequestedAt || null,
+                    },
+                    props: {
+                        isRoot: true,
+                    },
+                });
             },
-            props: {
-                isRoot: true,
+            order: async (lock) => {
+                logger.debug("Reissuing SSL certificate for domain: " + dashboardDomain.fullDomain, { fullDomain: dashboardDomain.fullDomain });
+                return await this.orderCert(dashboardDomain, {
+                    reason: CertificateOrderReason.Reissue,
+                    lock: lock,
+                });
             },
         });
-        if (claimedRowCount === 0) {
-            if (dashboardDomain.certificateReissueRequestedAt) {
-                throw new TooManyRequestsException(CertificateReissueUtil.getCooldownMessage(dashboardDomain.certificateReissueRequestedAt, now));
-            }
-            /*
-             * The row matched the cooldown a moment ago and does not now: another
-             * request claimed it in between, or the domain was deleted. Either way
-             * the caller must not order.
-             */
-            throw new BadDataException("Could not start a certificate reissue for this domain. Please refresh the page and try again.");
-        }
-        logger.debug("Reissuing SSL certificate for domain: " + dashboardDomain.fullDomain, { fullDomain: dashboardDomain.fullDomain });
-        await this.orderCert(dashboardDomain);
     }
     async updateSslProvisioningStatusForAllDomains() {
         const domains = await this.findBy({
@@ -280,6 +435,7 @@ export class Service extends DatabaseService {
             },
             select: {
                 _id: true,
+                fullDomain: true,
             },
             limit: LIMIT_MAX,
             skip: 0,
@@ -287,9 +443,17 @@ export class Service extends DatabaseService {
                 isRoot: true,
             },
         });
-        for (const domain of domains) {
-            await this.updateSslProvisioningStatus(domain);
-        }
+        await ArrayUtil.forEachWithConcurrency(domains, Service.DOMAIN_CHECK_CONCURRENCY, async (domain) => {
+            try {
+                await this.updateSslProvisioningStatus(domain);
+            }
+            catch (err) {
+                // one unreachable domain must not end the sweep for the rest.
+                logger.error(err, {
+                    fullDomain: domain.fullDomain,
+                });
+            }
+        });
     }
     async isSSLProvisioned(fulldomain, token) {
         try {
@@ -340,7 +504,16 @@ export class Service extends DatabaseService {
             });
         }
     }
-    async isCnameValid(fullDomain) {
+    async isCnameValid(fullDomain, options) {
+        const recordResult = async (cnameStatus) => {
+            if ((options === null || options === void 0 ? void 0 : options.recordResult) === false) {
+                return;
+            }
+            await this.updateCnameStatusForDashboardDomain({
+                domain: fullDomain,
+                cnameStatus: cnameStatus,
+            });
+        };
         try {
             logger.debug("Checking for CNAME " + fullDomain, {
                 fullDomain,
@@ -375,10 +548,7 @@ export class Service extends DatabaseService {
                 });
                 logger.debug(result, { fullDomain });
                 if (result.isSuccess()) {
-                    await this.updateCnameStatusForDashboardDomain({
-                        domain: fullDomain,
-                        cnameStatus: true,
-                    });
+                    await recordResult(true);
                     return true;
                 }
             }
@@ -401,10 +571,7 @@ export class Service extends DatabaseService {
                 });
                 logger.debug(resultHttps, { fullDomain });
                 if (resultHttps.isSuccess()) {
-                    await this.updateCnameStatusForDashboardDomain({
-                        domain: fullDomain,
-                        cnameStatus: true,
-                    });
+                    await recordResult(true);
                     return true;
                 }
             }
@@ -425,20 +592,14 @@ export class Service extends DatabaseService {
                     }
                     if (!cnameRecord) {
                         logger.debug(`No CNAME record found for ${fullDomain}. Expected record: ${DashboardCNameRecord}`, { fullDomain });
-                        await this.updateCnameStatusForDashboardDomain({
-                            domain: fullDomain,
-                            cnameStatus: false,
-                        });
+                        await recordResult(false);
                         return false;
                     }
                     if (cnameRecord &&
                         cnameRecord.trim().toLocaleLowerCase() ===
                             DashboardCNameRecord.trim().toLocaleLowerCase()) {
                         logger.debug(`CNAME record for ${fullDomain} matches the expected record: ${DashboardCNameRecord}`, { fullDomain });
-                        await this.updateCnameStatusForDashboardDomain({
-                            domain: fullDomain,
-                            cnameStatus: true,
-                        });
+                        await recordResult(true);
                         return true;
                     }
                     logger.debug(`CNAME record for ${fullDomain} is ${cnameRecord} and it does not match the expected record: ${DashboardCNameRecord}`, { fullDomain });
@@ -450,22 +611,30 @@ export class Service extends DatabaseService {
                 });
                 logger.debug(err, { fullDomain });
             }
-            await this.updateCnameStatusForDashboardDomain({
-                domain: fullDomain,
-                cnameStatus: false,
-            });
+            await recordResult(false);
             return false;
         }
         catch (err) {
             logger.debug("Failed checking for CNAME " + fullDomain);
             logger.debug(err);
-            await this.updateCnameStatusForDashboardDomain({
-                domain: fullDomain,
-                cnameStatus: false,
-            });
+            await recordResult(false);
             return false;
         }
     }
+    /*
+     * Record whether this domain's certificate is being served yet.
+     *
+     * Unlike the status page sweep this never orders a certificate. A probe
+     * that fails right after an order is usually racing nginx, which writes
+     * new certificates to disk every 15 minutes, and re-ordering then spends a
+     * second order on the same name. The two cases that do need an order have
+     * capped sweeps of their own: checkOrderStatus re-orders a certificate that
+     * has gone missing, and the renewal run renews one that is about to expire.
+     *
+     * When the probe fails the CNAME is checked again, so a domain whose record
+     * was removed goes back to asking for it. The result is written only when
+     * it changes: this sweep visits every ordered domain every 15 minutes.
+     */
     async updateSslProvisioningStatus(domain) {
         var _a;
         if (!domain.id) {
@@ -479,6 +648,7 @@ export class Service extends DatabaseService {
                 _id: true,
                 fullDomain: true,
                 cnameVerificationToken: true,
+                isSslProvisioned: true,
             },
             props: {
                 isRoot: true,
@@ -487,36 +657,17 @@ export class Service extends DatabaseService {
         if (!dashboardDomain) {
             throw new BadDataException("DomainModel not found");
         }
-        logger.debug(`DashboardCerts:RemoveCerts - Checking CNAME ${dashboardDomain.fullDomain}`, { fullDomain: dashboardDomain.fullDomain });
-        const isValid = await this.isSSLProvisioned(dashboardDomain.fullDomain, dashboardDomain.cnameVerificationToken);
-        if (!isValid) {
-            const isCnameValid = await this.isCnameValid(dashboardDomain.fullDomain);
-            await this.updateOneById({
-                id: dashboardDomain.id,
-                data: {
-                    isSslProvisioned: false,
-                },
-                props: {
-                    isRoot: true,
-                },
-            });
-            if (isCnameValid) {
-                try {
-                    await this.orderCert(dashboardDomain);
-                }
-                catch (err) {
-                    logger.error("Cannot order cert for domain: " + dashboardDomain.fullDomain, { fullDomain: dashboardDomain.fullDomain });
-                    logger.error(err, {
-                        fullDomain: dashboardDomain.fullDomain,
-                    });
-                }
-            }
+        logger.debug(`DashboardCerts:CheckSslProvisioningStatus - Checking ${dashboardDomain.fullDomain}`, { fullDomain: dashboardDomain.fullDomain });
+        const isProvisioned = await this.isSSLProvisioned(dashboardDomain.fullDomain, dashboardDomain.cnameVerificationToken);
+        if (!isProvisioned) {
+            // isCnameValid also records the result on the domain.
+            await this.isCnameValid(dashboardDomain.fullDomain);
         }
-        else {
+        if (Boolean(dashboardDomain.isSslProvisioned) !== isProvisioned) {
             await this.updateOneById({
                 id: dashboardDomain.id,
                 data: {
-                    isSslProvisioned: true,
+                    isSslProvisioned: isProvisioned,
                 },
                 props: {
                     isRoot: true,
@@ -524,40 +675,74 @@ export class Service extends DatabaseService {
             });
         }
     }
+    /*
+     * Order the first certificate of each of these domains, whose CNAME is
+     * verified: at most ORDER_MAX_PER_RUN orders in a window between the
+     * sweeps that order, picked afresh every run; a domain that has a
+     * certificate already is only recorded as ordered, and one whose orders
+     * keep failing waits longer after each failure
+     * (CustomDomainOrders.orderForVerifiedDomainsWithoutOne, which the status
+     * page sweeps share).
+     */
+    async orderCertsForVerifiedDomainsWithoutOne(domains, options) {
+        await CustomDomainOrders.orderForVerifiedDomainsWithoutOne({
+            domains: domains,
+            maxPerRun: Service.ORDER_MAX_PER_RUN,
+            recordAsOrdered: async (domainIds) => {
+                await this.recordCertificatesAsOrdered(domainIds);
+            },
+            orderIfMissing: async (domain) => {
+                return await this.orderCertIfMissing(domain, {
+                    fromSweep: true,
+                    cnameVerifiedJustNow: options === null || options === void 0 ? void 0 : options.cnameVerifiedJustNow,
+                });
+            },
+        });
+    }
+    /*
+     * Order the first certificate for domains whose CNAME is verified.
+     *
+     * Only verified domains: verifyCnameWhoseCnameisNotVerified checks the
+     * rest - and orders each one's certificate the moment it is verified, so
+     * this sweep is the fallback for the rest - and ordering one whose record
+     * is not in place would only fail the CNAME check again.
+     *
+     * A domain that already has a certificate is only recorded as ordered -
+     * all of them in one write. That is the domain whose CNAME check failed
+     * for a moment - which marks it unordered - and then passed again.
+     * Ordering for it would spend an order on a duplicate (Let's Encrypt
+     * allows five a week per name), and if its certificate is due, the renewal
+     * run orders it in this same tick. The rest are ordered through
+     * orderCertIfMissing, at most ORDER_MAX_PER_RUN in a window between the
+     * sweeps that order, picked afresh every run
+     * (GreenlockUtil.pickForThisRun), so a domain whose order keeps failing
+     * cannot hold a slot the others are waiting for; it also waits longer
+     * after each failure in a row (CertificateOrderFailures).
+     */
     async orderSSLForDomainsWhichAreNotOrderedYet() {
         return Telemetry.startActiveSpan({
             name: "DashboardDomainService.orderSSLForDomainsWhichAreNotOrderedYet",
             options: { attributes: {} },
             fn: async (span) => {
                 try {
-                    const domains = await this.findBy({
+                    const domains = await this.findAllBy({
                         query: {
+                            isCnameVerified: true,
                             isSslOrdered: false,
                             isCustomCertificate: false,
                         },
                         select: {
                             _id: true,
                             fullDomain: true,
+                            isSslOrdered: true,
+                            isCustomCertificate: true,
                         },
-                        limit: LIMIT_MAX,
                         skip: 0,
                         props: {
                             isRoot: true,
                         },
                     });
-                    for (const domain of domains) {
-                        try {
-                            logger.debug("Ordering SSL for domain: " + domain.fullDomain, {
-                                fullDomain: domain.fullDomain,
-                            });
-                            await this.orderCert(domain);
-                        }
-                        catch (e) {
-                            logger.error(e, {
-                                fullDomain: domain.fullDomain,
-                            });
-                        }
-                    }
+                    await this.orderCertsForVerifiedDomainsWithoutOne(domains);
                     Telemetry.endSpan(span);
                 }
                 catch (err) {
@@ -570,6 +755,15 @@ export class Service extends DatabaseService {
             },
         });
     }
+    /*
+     * Check every domain whose CNAME is not verified yet, at most
+     * DOMAIN_CHECK_CONCURRENCY at a time, and order the certificate of each
+     * one that is verified now straight away rather than at the next run of
+     * the order sweep - as the status page sweep does. So a domain is served
+     * on its own certificate within about 15 minutes of its record going live
+     * - nginx writes new certificates every 15 minutes - whether or not
+     * anyone clicked Check now.
+     */
     async verifyCnameWhoseCnameisNotVerified() {
         const domains = await this.findBy({
             query: {
@@ -578,6 +772,8 @@ export class Service extends DatabaseService {
             select: {
                 _id: true,
                 fullDomain: true,
+                isCustomCertificate: true,
+                isSslOrdered: true,
             },
             limit: LIMIT_MAX,
             skip: 0,
@@ -585,19 +781,84 @@ export class Service extends DatabaseService {
                 isRoot: true,
             },
         });
-        for (const domain of domains) {
+        const verifiedNow = [];
+        await ArrayUtil.forEachWithConcurrency(domains, Service.DOMAIN_CHECK_CONCURRENCY, async (domain) => {
             try {
-                await this.isCnameValid(domain.fullDomain);
+                // isCnameValid also records the result on the domain.
+                const isVerified = await this.isCnameValid(domain.fullDomain);
+                if (isVerified && !domain.isCustomCertificate) {
+                    verifiedNow.push(domain);
+                }
             }
             catch (e) {
                 logger.error(e, { fullDomain: domain.fullDomain });
             }
+        });
+        try {
+            /*
+             * Each was verified by this run a moment ago, so the orders do not
+             * check the CNAME again.
+             */
+            await this.orderCertsForVerifiedDomainsWithoutOne(verifiedNow, {
+                cnameVerifiedJustNow: true,
+            });
+        }
+        catch (e) {
+            // The order sweep orders them on its next run.
+            logger.error(e);
+        }
+    }
+    /*
+     * Which of these certificate domains are dashboard domains. The renewal
+     * run renews - and removes - only the certificates this returns, so the
+     * certificates of status page domains and of the primary host, which share
+     * the AcmeCertificate table, are never mistaken for dashboard domains
+     * whose CNAME stopped validating.
+     *
+     * A domain serving a certificate its owner uploaded still claims its
+     * Let's Encrypt certificate, which stays renewed meanwhile: nginx serves
+     * the uploaded one for that name (AcmeWriteCertificates leaves it alone),
+     * and switching back is then instant, on every nginx replica, including
+     * ones that start with an empty certificate directory.
+     */
+    async getOwnedDomains(domains) {
+        if (domains.length === 0) {
+            return [];
         }
+        const dashboardDomains = await this.findBy({
+            query: {
+                fullDomain: QueryHelper.any(domains),
+            },
+            select: {
+                fullDomain: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        return dashboardDomains
+            .map((dashboardDomain) => {
+            return dashboardDomain.fullDomain || "";
+        })
+            .filter((fullDomain) => {
+            return fullDomain.length > 0;
+        });
     }
     async renewCertsWhichAreExpiringSoon() {
         await GreenlockUtil.renewAllCertsWhichAreExpiringSoon({
+            maxPerRun: Service.ORDER_MAX_PER_RUN,
+            getOwnedDomains: async (domains) => {
+                return await this.getOwnedDomains(domains);
+            },
+            /*
+             * A check that writes nothing: a DNS blip while a certificate is
+             * renewed must not mark the domain unverified and unordered while its
+             * certificate keeps serving.
+             */
             validateCname: async (fullDomain) => {
-                return await this.isCnameValid(fullDomain);
+                return await this.isCnameValid(fullDomain, { recordResult: false });
             },
             notifyDomainRemoved: async (domain) => {
                 await this.updateOneBy({
@@ -618,8 +879,21 @@ export class Service extends DatabaseService {
             },
         });
     }
+    /*
+     * Re-order the certificate of a domain that says it has one ordered but
+     * whose certificate is gone, at most ORDER_MAX_PER_RUN per run.
+     *
+     * This is how dashboard domains recover from the status page renewal job,
+     * which until it learned to check ownership deleted every dashboard
+     * certificate that came due for renewal while the domain kept saying
+     * "ordered" - nginx then served the stale copy on disk until it expired.
+     * Domains are picked afresh every run, for the same reason as in the
+     * first-order sweep above, and re-ordered through orderCertIfMissing, so a
+     * re-order never races another order for the same name and shares the
+     * sweeps' budget.
+     */
     async checkOrderStatus() {
-        const domains = await this.findBy({
+        const domains = await this.findAllBy({
             query: {
                 isSslOrdered: true,
                 isCustomCertificate: false,
@@ -627,43 +901,83 @@ export class Service extends DatabaseService {
             select: {
                 _id: true,
                 fullDomain: true,
-                cnameVerificationToken: true,
+                isCustomCertificate: true,
             },
-            limit: LIMIT_MAX,
             skip: 0,
             props: {
                 isRoot: true,
             },
         });
-        for (const domain of domains) {
-            if (!domain.fullDomain) {
-                continue;
+        const fullDomains = domains
+            .map((domain) => {
+            return domain.fullDomain || "";
+        })
+            .filter((fullDomain) => {
+            return fullDomain.length > 0;
+        });
+        if (fullDomains.length === 0) {
+            return;
+        }
+        const certificates = await GreenlockUtil.findCertificatesByDomain(fullDomains.map((fullDomain) => {
+            return CertificateOrder.normalizeDomain(fullDomain);
+        }));
+        const now = OneUptimeDate.getCurrentDate();
+        const domainsWithoutCertificate = await CertificateOrderFailures.withoutThoseWaitingToRetry({
+            items: domains.filter((domain) => {
+                return (Boolean(domain.id) &&
+                    Boolean(domain.fullDomain) &&
+                    !certificates.has(CertificateOrder.normalizeDomain(domain.fullDomain)));
+            }),
+            getDomain: (domain) => {
+                return domain.fullDomain;
+            },
+            now: now,
+        });
+        const batch = GreenlockUtil.pickForThisRun({
+            items: domainsWithoutCertificate,
+            max: Service.ORDER_MAX_PER_RUN,
+            now: now,
+            getKey: (domain) => {
+                return domain.fullDomain || "";
+            },
+        });
+        for (const domain of batch) {
+            try {
+                await this.orderCertIfMissing(domain, { fromSweep: true });
             }
-            const acmeCert = await AcmeCertificateService.findOneBy({
-                query: {
-                    domain: domain.fullDomain,
-                },
-                select: {
-                    _id: true,
-                },
-                props: {
-                    isRoot: true,
-                },
-            });
-            if (!acmeCert) {
-                try {
-                    await this.orderCert(domain);
-                }
-                catch (err) {
-                    logger.error("Cannot order cert for domain: " + domain.fullDomain, {
-                        fullDomain: domain.fullDomain,
-                    });
-                    logger.error(err, { fullDomain: domain.fullDomain });
-                }
+            catch (err) {
+                logger.error("Cannot order cert for domain: " + domain.fullDomain, {
+                    fullDomain: domain.fullDomain,
+                });
+                logger.error(err, { fullDomain: domain.fullDomain });
             }
         }
     }
 }
+/*
+ * How many certificates the dashboard sweeps may order: the first orders
+ * and the re-orders of a certificate that has gone missing together, in
+ * one 15-minute window (SWEEP_ORDER_BUDGET, CertificateOrder.takeOrderSlot)
+ * - and, separately, how many the dashboard renewal run renews per run.
+ *
+ * Every order spends from the one Let's Encrypt account the whole
+ * installation shares - 300 new orders per account per three hours - and
+ * status page certificates, the larger fleet, are ordered and renewed from
+ * it too. These caps keep a backlog - such as the first runs re-ordering
+ * every dashboard certificate the status page renewal job used to delete -
+ * worked off a few domains per run rather than in one burst, and none of
+ * them can order beyond the installation's own budget for that account
+ * (CertificateOrderBudget), which every order of every kind draws from.
+ */
+Service.ORDER_MAX_PER_RUN = 5;
+// The budget the dashboard ordering sweeps share (CertificateOrder.takeOrderSlot).
+Service.SWEEP_ORDER_BUDGET = "DashboardDomainSweeps";
+/*
+ * How many domains the verification and provisioning sweeps check at once.
+ * Each check is a bounded request out to a customer domain; the same
+ * figure the status page provisioning sweep uses.
+ */
+Service.DOMAIN_CHECK_CONCURRENCY = 10;
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
@@ -697,9 +1011,27 @@ __decorate([
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [DashboardDomain]),
+    __metadata("design:paramtypes", [DashboardDomain, Object]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "orderCert", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [DashboardDomain, Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "orderCertIfMissing", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [DashboardDomain, Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "orderCertOnceCnameIsVerified", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getCertificates", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
@@ -721,7 +1053,7 @@ __decorate([
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String]),
+    __metadata("design:paramtypes", [String, Object]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "isCnameValid", null);
 __decorate([
@@ -742,6 +1074,12 @@ __decorate([
     __metadata("design:paramtypes", []),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "verifyCnameWhoseCnameisNotVerified", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getOwnedDomains", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),