@oneuptime/common 14.0.13 → 14.0.14

package/build/dist/Server/Utils/Greenlock/CertificateOrderFailures.js

@@ -0,0 +1,208 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import GlobalCache from "../../Infrastructure/GlobalCache";
+import CertificateOrderLock from "./CertificateOrderLock";
+import logger from "../Logger";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+import OneUptimeDate from "../../../Types/Date";
+/*
+ * The last failed certificate order of each name, kept in Redis.
+ *
+ * An order that fails leaves nothing behind in the database: the domain
+ * stays unordered, and the sweeps order it again. So a domain whose order
+ * keeps failing - a CAA record that leaves Let's Encrypt out, a server Let's
+ * Encrypt cannot reach - said "Issuing a free certificate" for days, and the
+ * reason was only in the worker's log. GreenlockUtil.orderCert records every
+ * failed order here and clears the record when an order for the name
+ * succeeds, so:
+ *
+ *   - the domain's Status column shows that the order failed, and why;
+ *   - the sweeps and renewal runs, which would otherwise order a failing name
+ *     again every 15 minutes, wait longer after each failure in a row
+ *     (getRetryDelayInMinutes). Every failed order still counts against the
+ *     installation's shared Let's Encrypt account, and Let's Encrypt allows
+ *     five failed validations per name per hour.
+ *
+ * Check now and the order API do not wait: somebody who has just fixed the
+ * cause should not have to. They have a window of their own per domain
+ * (CertificateOrder.claimOnDemandOrder).
+ *
+ * Kept in Redis rather than on the domain row, so neither table changes:
+ * losing a record (a Redis restart) only means the next failure is recorded
+ * again. Every method here never throws.
+ */
+class CertificateOrderFailures {
+    static getRetryDelayInMinutes(failures) {
+        const failuresInARow = Math.max(1, Math.floor(failures || 1));
+        const longest = CertificateOrderFailures.LONGEST_RETRY_DELAY_IN_HOURS * 60;
+        // Past this many doublings the delay is the longest one anyway.
+        if (failuresInARow > 20) {
+            return longest;
+        }
+        return Math.min(CertificateOrderFailures.FIRST_RETRY_DELAY_IN_MINUTES *
+            Math.pow(2, failuresInARow - 1), longest);
+    }
+    // When the sweeps and renewal runs order this name again.
+    static getRetryAt(failure) {
+        return OneUptimeDate.addRemoveMinutes(failure.failedAt, CertificateOrderFailures.getRetryDelayInMinutes(failure.failures));
+    }
+    static isWaitingToRetry(failure, now) {
+        if (!failure) {
+            return false;
+        }
+        return OneUptimeDate.isAfter(OneUptimeDate.addRemoveMinutes(CertificateOrderFailures.getRetryAt(failure), -CertificateOrderFailures.RETRY_GRACE_IN_MINUTES), now);
+    }
+    static async record(data) {
+        const name = CertificateOrderFailures.normalizeDomain(data.domain);
+        try {
+            const previous = CertificateOrderFailures.parse(await GlobalCache.getString(CertificateOrderFailures.NAMESPACE, name));
+            const failure = {
+                error: data.error.trim() ||
+                    "We could not order an SSL certificate for this domain.",
+                failedAt: data.now,
+                failures: ((previous === null || previous === void 0 ? void 0 : previous.failures) || 0) + 1,
+            };
+            await GlobalCache.setString(CertificateOrderFailures.NAMESPACE, name, JSON.stringify({
+                error: failure.error,
+                failedAt: failure.failedAt.toISOString(),
+                failures: failure.failures,
+            }), {
+                expiresInSeconds: OneUptimeDate.getSecondsInDays(CertificateOrderFailures.KEEP_FOR_IN_DAYS),
+            });
+        }
+        catch (err) {
+            logger.error(`Could not record the failed certificate order of ${name}`, {
+                fullDomain: name,
+            });
+            logger.error(err, { fullDomain: name });
+        }
+    }
+    // An order for the name succeeded: forget its failures.
+    static async clear(domain) {
+        const name = CertificateOrderFailures.normalizeDomain(domain);
+        try {
+            await GlobalCache.deleteKey(CertificateOrderFailures.NAMESPACE, name);
+        }
+        catch (err) {
+            logger.error(err, { fullDomain: name });
+        }
+    }
+    /*
+     * The recorded failure of each of these names that has one, by normalized
+     * name, a few hundred names per round trip. Empty when Redis cannot be
+     * read: then nothing waits, and nothing shows a failure.
+     */
+    static async get(domains) {
+        const failures = new Map();
+        const names = Array.from(new Set(domains
+            .map((domain) => {
+            return CertificateOrderFailures.normalizeDomain(domain);
+        })
+            .filter((name) => {
+            return name.length > 0;
+        })));
+        try {
+            for (let offset = 0; offset < names.length; offset += CertificateOrderFailures.LOOKUP_CHUNK_SIZE) {
+                const chunk = names.slice(offset, offset + CertificateOrderFailures.LOOKUP_CHUNK_SIZE);
+                const values = await GlobalCache.getStrings(CertificateOrderFailures.NAMESPACE, chunk);
+                chunk.forEach((name, index) => {
+                    var _a;
+                    const failure = CertificateOrderFailures.parse((_a = values[index]) !== null && _a !== void 0 ? _a : null);
+                    if (failure) {
+                        failures.set(name, failure);
+                    }
+                });
+            }
+        }
+        catch (err) {
+            logger.error("Could not read the failed certificate orders");
+            logger.error(err);
+            return new Map();
+        }
+        return failures;
+    }
+    /*
+     * The items whose name is not waiting after a failed order - what a sweep
+     * or a renewal run may order now.
+     */
+    static async withoutThoseWaitingToRetry(data) {
+        if (data.items.length === 0) {
+            return [];
+        }
+        const failures = await CertificateOrderFailures.get(data.items.map(data.getDomain));
+        return data.items.filter((item) => {
+            return !CertificateOrderFailures.isWaitingToRetry(failures.get(CertificateOrderFailures.normalizeDomain(data.getDomain(item))), data.now);
+        });
+    }
+    // The same name the order lock and the certificate table use.
+    static normalizeDomain(domain) {
+        return CertificateOrderLock.normalizeDomain(domain || "");
+    }
+    static parse(value) {
+        if (!value) {
+            return null;
+        }
+        try {
+            const json = JSON.parse(value);
+            const failedAt = new Date(String(json.failedAt));
+            if (typeof json.error !== "string" || isNaN(failedAt.getTime())) {
+                return null;
+            }
+            const failures = Number(json.failures);
+            return {
+                error: json.error,
+                failedAt: failedAt,
+                failures: Number.isFinite(failures) && failures > 0 ? failures : 1,
+            };
+        }
+        catch (_a) {
+            return null;
+        }
+    }
+}
+CertificateOrderFailures.NAMESPACE = "CustomDomainCertificateOrderFailure";
+/*
+ * How long a record is kept without another failure. The sweeps retry a
+ * failing name at least every LONGEST_RETRY_DELAY_IN_HOURS, which renews
+ * it well before this.
+ */
+CertificateOrderFailures.KEEP_FOR_IN_DAYS = 7;
+// How long the automatic attempts wait after one failure...
+CertificateOrderFailures.FIRST_RETRY_DELAY_IN_MINUTES = 15;
+// ...doubling with each further failure in a row, up to this.
+CertificateOrderFailures.LONGEST_RETRY_DELAY_IN_HOURS = 6;
+/*
+ * A failure is recorded when the order ends, which can be a minute or two
+ * after the sweep started it. Without this, the sweep on the tick the
+ * delay ends on would find the name still waiting, and the first retry
+ * would come a tick late.
+ */
+CertificateOrderFailures.RETRY_GRACE_IN_MINUTES = 2;
+CertificateOrderFailures.LOOKUP_CHUNK_SIZE = 500;
+export default CertificateOrderFailures;
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], CertificateOrderFailures, "record", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String]),
+    __metadata("design:returntype", Promise)
+], CertificateOrderFailures, "clear", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array]),
+    __metadata("design:returntype", Promise)
+], CertificateOrderFailures, "get", null);
+//# sourceMappingURL=CertificateOrderFailures.js.map

package/build/dist/Server/Utils/Greenlock/CertificateOrderFailures.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CertificateOrderFailures.js","sourceRoot":"","sources":["../../../../../Server/Utils/Greenlock/CertificateOrderFailures.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAC3D,OAAO,oBAAoB,MAAM,wBAAwB,CAAC;AAC1D,OAAO,MAAyB,MAAM,WAAW,CAAC;AAClD,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAUhD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAqB,wBAAwB;IA2BpC,MAAM,CAAC,sBAAsB,CAAC,QAAgB;QACnD,MAAM,cAAc,GAAW,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC;QAEtE,MAAM,OAAO,GACX,wBAAwB,CAAC,4BAA4B,GAAG,EAAE,CAAC;QAE7D,gEAAgE;QAChE,IAAI,cAAc,GAAG,EAAE,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,OAAO,IAAI,CAAC,GAAG,CACb,wBAAwB,CAAC,4BAA4B;YACnD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,GAAG,CAAC,CAAC,EACjC,OAAO,CACR,CAAC;IACJ,CAAC;IAED,0DAA0D;IACnD,MAAM,CAAC,UAAU,CAAC,OAAgC;QACvD,OAAO,aAAa,CAAC,gBAAgB,CACnC,OAAO,CAAC,QAAQ,EAChB,wBAAwB,CAAC,sBAAsB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAClE,CAAC;IACJ,CAAC;IAEM,MAAM,CAAC,gBAAgB,CAC5B,OAA4C,EAC5C,GAAS;QAET,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,aAAa,CAAC,OAAO,CAC1B,aAAa,CAAC,gBAAgB,CAC5B,wBAAwB,CAAC,UAAU,CAAC,OAAO,CAAC,EAC5C,CAAC,wBAAwB,CAAC,sBAAsB,CACjD,EACD,GAAG,CACJ,CAAC;IACJ,CAAC;IAGmB,AAAb,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAI1B;QACC,MAAM,IAAI,GAAW,wBAAwB,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE3E,IAAI,CAAC;YACH,MAAM,QAAQ,GACZ,wBAAwB,CAAC,KAAK,CAC5B,MAAM,WAAW,CAAC,SAAS,CAAC,wBAAwB,CAAC,SAAS,EAAE,IAAI,CAAC,CACtE,CAAC;YAEJ,MAAM,OAAO,GAA4B;gBACvC,KAAK,EACH,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;oBACjB,wDAAwD;gBAC1D,QAAQ,EAAE,IAAI,CAAC,GAAG;gBAClB,QAAQ,EAAE,CAAC,CAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,QAAQ,KAAI,CAAC,CAAC,GAAG,CAAC;aACxC,CAAC;YAEF,MAAM,WAAW,CAAC,SAAS,CACzB,wBAAwB,CAAC,SAAS,EAClC,IAAI,EACJ,IAAI,CAAC,SAAS,CAAC;gBACb,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE;gBACxC,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B,CAAC,EACF;gBACE,gBAAgB,EAAE,aAAa,CAAC,gBAAgB,CAC9C,wBAAwB,CAAC,gBAAgB,CAC1C;aACF,CACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,oDAAoD,IAAI,EAAE,EAAE;gBACvE,UAAU,EAAE,IAAI;aACA,CAAC,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAmB,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,wDAAwD;IAEpC,AAAb,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAc;QACtC,MAAM,IAAI,GAAW,wBAAwB,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAEtE,IAAI,CAAC;YACH,MAAM,WAAW,CAAC,SAAS,CAAC,wBAAwB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACxE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAmB,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED;;;;OAIG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,GAAG,CACrB,OAAsB;QAEtB,MAAM,QAAQ,GAAyC,IAAI,GAAG,EAG3D,CAAC;QAEJ,MAAM,KAAK,GAAkB,KAAK,CAAC,IAAI,CACrC,IAAI,GAAG,CACL,OAAO;aACJ,GAAG,CAAC,CAAC,MAAc,EAAE,EAAE;YACtB,OAAO,wBAAwB,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAC1D,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAY,EAAE,EAAE;YACvB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACzB,CAAC,CAAC,CACL,CACF,CAAC;QAEF,IAAI,CAAC;YACH,KACE,IAAI,MAAM,GAAW,CAAC,EACtB,MAAM,GAAG,KAAK,CAAC,MAAM,EACrB,MAAM,IAAI,wBAAwB,CAAC,iBAAiB,EACpD,CAAC;gBACD,MAAM,KAAK,GAAkB,KAAK,CAAC,KAAK,CACtC,MAAM,EACN,MAAM,GAAG,wBAAwB,CAAC,iBAAiB,CACpD,CAAC;gBAEF,MAAM,MAAM,GAAyB,MAAM,WAAW,CAAC,UAAU,CAC/D,wBAAwB,CAAC,SAAS,EAClC,KAAK,CACN,CAAC;gBAEF,KAAK,CAAC,OAAO,CAAC,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE;;oBAC5C,MAAM,OAAO,GACX,wBAAwB,CAAC,KAAK,CAAC,MAAA,MAAM,CAAC,KAAK,CAAC,mCAAI,IAAI,CAAC,CAAC;oBAExD,IAAI,OAAO,EAAE,CAAC;wBACZ,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;oBAC9B,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAC7D,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAElB,OAAO,IAAI,GAAG,EAAmC,CAAC;QACpD,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAI,IAIjD;QACC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,QAAQ,GACZ,MAAM,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QAErE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAO,EAAE,EAAE;YACnC,OAAO,CAAC,wBAAwB,CAAC,gBAAgB,CAC/C,QAAQ,CAAC,GAAG,CACV,wBAAwB,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAC/D,EACD,IAAI,CAAC,GAAG,CACT,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,8DAA8D;IACvD,MAAM,CAAC,eAAe,CAAC,MAAc;QAC1C,OAAO,oBAAoB,CAAC,eAAe,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,KAAoB;QACvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAIN,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAEtB,MAAM,QAAQ,GAAS,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAEvD,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBAChE,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,QAAQ,GAAW,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAE/C,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,QAAQ;gBAClB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aACnE,CAAC;QACJ,CAAC;QAAC,WAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;;AArPsB,kCAAS,GAC9B,qCAAqC,CAAC;AAExC;;;;GAIG;AACoB,yCAAgB,GAAW,CAAC,CAAC;AAEpD,4DAA4D;AACrC,qDAA4B,GAAW,EAAE,CAAC;AAEjE,8DAA8D;AACvC,qDAA4B,GAAW,CAAC,CAAC;AAEhE;;;;;GAKG;AACoB,+CAAsB,GAAW,CAAC,CAAC;AAElC,0CAAiB,GAAW,GAAG,CAAC;eAzBrC,wBAAwB;AAuEvB;IADnB,WAAW,EAAE;;;;4CA0Cb;AAImB;IADnB,WAAW,EAAE;;;;2CASb;AAQmB;IADnB,WAAW,EAAE;;qCAEH,KAAK;;yCAoDf"}

package/build/dist/Server/Utils/Greenlock/CertificateOrderLock.js

@@ -0,0 +1,111 @@
+import Semaphore, { SemaphoreLockTimeoutError, } from "../../Infrastructure/Semaphore";
+import logger from "../Logger";
+import OneUptimeDate from "../../../Types/Date";
+/*
+ * One certificate order per name at a time, across every replica.
+ *
+ * A name's certificate can be ordered from many places: Check now, the
+ * sweeps that order first certificates and re-order missing ones, the order
+ * and reissue APIs, the renewal runs and, for the installation's own host,
+ * CoreSSL. Two orders for one name at once are two orders against the one
+ * Let's Encrypt account the whole installation shares, and two http-01
+ * challenges for one name, whose clean-up deletes the challenge rows by
+ * name - each order can remove the other's challenge while Let's Encrypt is
+ * still checking it.
+ *
+ * GreenlockUtil.orderCert takes this lock for every order, or checks that its
+ * caller holds it (CertificateOrder.orderIfMissing takes it first, to look
+ * the name up again before deciding to order). The lock is taken without
+ * waiting: a caller that finds it taken orders nothing, because the order
+ * already running is the one it would have placed.
+ */
+class CertificateOrderLock {
+    static normalizeDomain(domain) {
+        return domain.trim().toLowerCase();
+    }
+    /*
+     * The lock on the name, or null when it is taken - or cannot be taken at
+     * all. Without Redis nothing is ordered: the sweeps run from Redis' own
+     * job queue, and ordering without the lock is what this is here to stop.
+     */
+    static async tryLock(domain) {
+        const name = CertificateOrderLock.normalizeDomain(domain);
+        const handle = Object.freeze({
+            domain: name,
+        });
+        try {
+            const mutex = await Semaphore.lock({
+                key: name,
+                namespace: CertificateOrderLock.NAMESPACE,
+                lockTimeout: CertificateOrderLock.TIMEOUT_IN_MS,
+                acquireAttemptsLimit: 1,
+                onLockLost: (err) => {
+                    // From now on this handle holds nothing (isHeldFor says so).
+                    CertificateOrderLock.held.delete(handle);
+                    CertificateOrderLock.lost.add(handle);
+                    logger.error(`Lost the certificate order lock of ${name} while ordering`, { fullDomain: name });
+                    logger.error(err, { fullDomain: name });
+                },
+            });
+            CertificateOrderLock.held.set(handle, mutex);
+            return handle;
+        }
+        catch (err) {
+            if (err instanceof SemaphoreLockTimeoutError) {
+                logger.debug(`A certificate for ${name} is being ordered already: not ordering another`, { fullDomain: name });
+            }
+            else {
+                logger.error(`Could not take the certificate order lock of ${name}: not ordering now`, { fullDomain: name });
+                logger.error(err, { fullDomain: name });
+            }
+            return null;
+        }
+    }
+    // Whether this handle holds the lock of this name right now.
+    static isHeldFor(lock, domain) {
+        return Boolean(lock &&
+            CertificateOrderLock.held.has(lock) &&
+            lock.domain === CertificateOrderLock.normalizeDomain(domain));
+    }
+    /*
+     * Whether this handle did hold the name's lock, and Redis lost it while it
+     * was held: not a mistake of the caller's, just a lock that is gone.
+     */
+    static wasLost(lock, domain) {
+        return Boolean(lock &&
+            CertificateOrderLock.lost.has(lock) &&
+            lock.domain === CertificateOrderLock.normalizeDomain(domain));
+    }
+    // Never throws: the lock expires on its own, and an order must not fail on it.
+    static async release(lock) {
+        const mutex = CertificateOrderLock.held.get(lock);
+        if (!mutex) {
+            return;
+        }
+        CertificateOrderLock.held.delete(lock);
+        try {
+            await Semaphore.release(mutex);
+        }
+        catch (err) {
+            logger.error(err, { fullDomain: lock.domain });
+        }
+    }
+}
+CertificateOrderLock.NAMESPACE = "CustomDomainCertificateOrder";
+/*
+ * How long the lock outlives a holder that died without releasing it. A
+ * live holder refreshes it (redis-semaphore does, every 80% of this), so
+ * an order that takes longer - the CA polls a challenge for a while - keeps
+ * it.
+ */
+CertificateOrderLock.TIMEOUT_IN_MS = OneUptimeDate.convertMinutesToMilliseconds(2);
+// The handles tryLock gave out and that are not released yet.
+CertificateOrderLock.held = new WeakMap();
+/*
+ * Handles whose lock Redis lost while they held it - a refresh that
+ * failed, an eviction. Another replica can take the name's lock from then
+ * on, so such a handle holds nothing any more.
+ */
+CertificateOrderLock.lost = new WeakSet();
+export default CertificateOrderLock;
+//# sourceMappingURL=CertificateOrderLock.js.map

package/build/dist/Server/Utils/Greenlock/CertificateOrderLock.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CertificateOrderLock.js","sourceRoot":"","sources":["../../../../../Server/Utils/Greenlock/CertificateOrderLock.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,EAAE,EAChB,yBAAyB,GAE1B,MAAM,gCAAgC,CAAC;AACxC,OAAO,MAAyB,MAAM,WAAW,CAAC;AAClD,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAWhD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAqB,oBAAoB;IA0BhC,MAAM,CAAC,eAAe,CAAC,MAAc;QAC1C,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,KAAK,CAAC,OAAO,CACzB,MAAc;QAEd,MAAM,IAAI,GAAW,oBAAoB,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAElE,MAAM,MAAM,GAA+B,MAAM,CAAC,MAAM,CAAC;YACvD,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,KAAK,GAAmB,MAAM,SAAS,CAAC,IAAI,CAAC;gBACjD,GAAG,EAAE,IAAI;gBACT,SAAS,EAAE,oBAAoB,CAAC,SAAS;gBACzC,WAAW,EAAE,oBAAoB,CAAC,aAAa;gBAC/C,oBAAoB,EAAE,CAAC;gBACvB,UAAU,EAAE,CAAC,GAAU,EAAE,EAAE;oBACzB,6DAA6D;oBAC7D,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBACzC,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAEtC,MAAM,CAAC,KAAK,CACV,sCAAsC,IAAI,iBAAiB,EAC3D,EAAE,UAAU,EAAE,IAAI,EAAmB,CACtC,CAAC;oBACF,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAmB,CAAC,CAAC;gBAC3D,CAAC;aACF,CAAC,CAAC;YAEH,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAE7C,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,yBAAyB,EAAE,CAAC;gBAC7C,MAAM,CAAC,KAAK,CACV,qBAAqB,IAAI,iDAAiD,EAC1E,EAAE,UAAU,EAAE,IAAI,EAAmB,CACtC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,KAAK,CACV,gDAAgD,IAAI,oBAAoB,EACxE,EAAE,UAAU,EAAE,IAAI,EAAmB,CACtC,CAAC;gBACF,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAmB,CAAC,CAAC;YAC3D,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,6DAA6D;IACtD,MAAM,CAAC,SAAS,CACrB,IAA4C,EAC5C,MAAc;QAEd,OAAO,OAAO,CACZ,IAAI;YACF,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YACnC,IAAI,CAAC,MAAM,KAAK,oBAAoB,CAAC,eAAe,CAAC,MAAM,CAAC,CAC/D,CAAC;IACJ,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,OAAO,CACnB,IAA4C,EAC5C,MAAc;QAEd,OAAO,OAAO,CACZ,IAAI;YACF,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YACnC,IAAI,CAAC,MAAM,KAAK,oBAAoB,CAAC,eAAe,CAAC,MAAM,CAAC,CAC/D,CAAC;IACJ,CAAC;IAED,+EAA+E;IACxE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAgC;QAC1D,MAAM,KAAK,GACT,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEtC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO;QACT,CAAC;QAED,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvC,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAmB,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;;AA9HsB,8BAAS,GAAW,8BAA8B,CAAC;AAE1E;;;;;GAKG;AACoB,kCAAa,GAClC,aAAa,CAAC,4BAA4B,CAAC,CAAC,CAAC,CAAC;AAEhD,8DAA8D;AACtC,yBAAI,GAGxB,IAAI,OAAO,EAA8C,CAAC;AAE9D;;;;GAIG;AACqB,yBAAI,GAC1B,IAAI,OAAO,EAA8B,CAAC;eAxBzB,oBAAoB"}

package/build/dist/Server/Utils/Greenlock/CertificateOrderOutcome.js

@@ -0,0 +1,30 @@
+/*
+ * How a request to order a certificate ended, when it did not throw.
+ *
+ * Its own module so that GreenlockUtil, which places every order, and
+ * CertificateOrder, which decides whether a first order is needed, can both
+ * use it without importing each other.
+ */
+export var CertificateOrderOutcome;
+(function (CertificateOrderOutcome) {
+    // A certificate was ordered and stored.
+    CertificateOrderOutcome["Ordered"] = "Ordered";
+    /*
+     * The name already had a certificate. It was recorded as ordered and
+     * nothing was ordered.
+     */
+    CertificateOrderOutcome["AlreadyIssued"] = "AlreadyIssued";
+    /*
+     * Nothing was ordered now: another order for the name was running, or the
+     * lock that keeps orders apart could not be taken. The sweeps try again.
+     */
+    CertificateOrderOutcome["NotOrderedNow"] = "NotOrderedNow";
+    /*
+     * Nothing was ordered now: this window's orders are used up - the share of
+     * the sweep or the click that asked, or the installation's Let's Encrypt
+     * allowance (CertificateOrderBudget). The next window has new ones.
+     */
+    CertificateOrderOutcome["LimitReached"] = "LimitReached";
+})(CertificateOrderOutcome || (CertificateOrderOutcome = {}));
+export default CertificateOrderOutcome;
+//# sourceMappingURL=CertificateOrderOutcome.js.map

package/build/dist/Server/Utils/Greenlock/CertificateOrderOutcome.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CertificateOrderOutcome.js","sourceRoot":"","sources":["../../../../../Server/Utils/Greenlock/CertificateOrderOutcome.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,CAAN,IAAY,uBAmBX;AAnBD,WAAY,uBAAuB;IACjC,wCAAwC;IACxC,8CAAmB,CAAA;IACnB;;;OAGG;IACH,0DAA+B,CAAA;IAC/B;;;OAGG;IACH,0DAA+B,CAAA;IAC/B;;;;OAIG;IACH,wDAA6B,CAAA;AAC/B,CAAC,EAnBW,uBAAuB,KAAvB,uBAAuB,QAmBlC;AAED,eAAe,uBAAuB,CAAC"}

package/build/dist/Server/Utils/Greenlock/CertificateOwners.js

@@ -0,0 +1,52 @@
+import { Host } from "../../EnvironmentConfig";
+import DashboardDomainService from "../../Services/DashboardDomainService";
+import StatusPageDomainService from "../../Services/StatusPageDomainService";
+/*
+ * Every kind of owner of the certificates in AcmeCertificate.
+ *
+ * Each owner renews and removes its own certificates through
+ * GreenlockUtil.renewAllCertsWhichAreExpiringSoon with its own lookup; this
+ * list is for the one job that has to know about all of them at once, the
+ * cleanup of certificates nobody owns
+ * (GreenlockUtil.removeExpiredCertificatesNobodyOwns).
+ *
+ * A new kind of custom domain that orders certificates must be added here.
+ * Until it is, the cleanup takes its certificates for abandoned ones and
+ * deletes them a month after they expire.
+ */
+export default class CertificateOwners {
+    // The installation's own HOST, without a port, as CoreSSL orders it.
+    static getPrimaryHostname(host = Host) {
+        return (host.trim().toLowerCase().split(":")[0] || "").trim();
+    }
+    static getAll() {
+        return [
+            {
+                name: "status page domains",
+                getOwnedDomains: async (domains) => {
+                    return await StatusPageDomainService.getOwnedDomains(domains);
+                },
+            },
+            {
+                name: "dashboard domains",
+                getOwnedDomains: async (domains) => {
+                    return await DashboardDomainService.getOwnedDomains(domains);
+                },
+            },
+            {
+                // Ordered and renewed by the CoreSSL job.
+                name: "primary host",
+                getOwnedDomains: async (domains) => {
+                    const primaryHostname = CertificateOwners.getPrimaryHostname();
+                    if (!primaryHostname) {
+                        return [];
+                    }
+                    return domains.filter((domain) => {
+                        return domain === primaryHostname;
+                    });
+                },
+            },
+        ];
+    }
+}
+//# sourceMappingURL=CertificateOwners.js.map

package/build/dist/Server/Utils/Greenlock/CertificateOwners.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CertificateOwners.js","sourceRoot":"","sources":["../../../../../Server/Utils/Greenlock/CertificateOwners.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,sBAAsB,MAAM,uCAAuC,CAAC;AAC3E,OAAO,uBAAuB,MAAM,wCAAwC,CAAC;AAY7E;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,OAAO,OAAO,iBAAiB;IACpC,qEAAqE;IAC9D,MAAM,CAAC,kBAAkB,CAAC,OAAe,IAAI;QAClD,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAChE,CAAC;IAEM,MAAM,CAAC,MAAM;QAClB,OAAO;YACL;gBACE,IAAI,EAAE,qBAAqB;gBAC3B,eAAe,EAAE,KAAK,EACpB,OAAsB,EACE,EAAE;oBAC1B,OAAO,MAAM,uBAAuB,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;gBAChE,CAAC;aACF;YACD;gBACE,IAAI,EAAE,mBAAmB;gBACzB,eAAe,EAAE,KAAK,EACpB,OAAsB,EACE,EAAE;oBAC1B,OAAO,MAAM,sBAAsB,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;gBAC/D,CAAC;aACF;YACD;gBACE,0CAA0C;gBAC1C,IAAI,EAAE,cAAc;gBACpB,eAAe,EAAE,KAAK,EACpB,OAAsB,EACE,EAAE;oBAC1B,MAAM,eAAe,GACnB,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;oBAEzC,IAAI,CAAC,eAAe,EAAE,CAAC;wBACrB,OAAO,EAAE,CAAC;oBACZ,CAAC;oBAED,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,MAAc,EAAE,EAAE;wBACvC,OAAO,MAAM,KAAK,eAAe,CAAC;oBACpC,CAAC,CAAC,CAAC;gBACL,CAAC;aACF;SACF,CAAC;IACJ,CAAC;CACF"}

package/build/dist/Server/Utils/Greenlock/CertificateReissueOrder.js

@@ -0,0 +1,80 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import CertificateOrderLock from "./CertificateOrderLock";
+import { CertificateOrderOutcome } from "./CertificateOrderOutcome";
+import logger from "../Logger";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+import CertificateReissueUtil from "../../../Utils/CertificateReissue";
+import BadDataException from "../../../Types/Exception/BadDataException";
+import TooManyRequestsException from "../../../Types/Exception/TooManyRequestsException";
+/*
+ * Reissue SSL - a brand new certificate a customer asked for - for status
+ * page and dashboard domains alike, once the domain's own service has found
+ * it eligible.
+ *
+ * In this order:
+ *   1. The name's order lock (CertificateOrderLock). A reissue used to order
+ *      without it, so it could run beside Check now, a sweep or a renewal
+ *      ordering the same name: two orders, and two http-01 challenges whose
+ *      clean-up removes each other's challenge rows. A reissue that finds
+ *      the lock taken is refused, and its cooldown is not touched.
+ *   2. The cooldown, claimed by a conditional write. It is never given back
+ *      when the order fails: a rejected order still costs a validation
+ *      against the shared account, and a domain whose order fails is exactly
+ *      the one somebody presses again.
+ *   3. The order, under the lock. When the installation's Let's Encrypt
+ *      allowance for the window is used up (CertificateOrderBudget), nothing
+ *      is ordered, so the cooldown is given back and the customer is told to
+ *      try again shortly.
+ */
+export default class CertificateReissueOrder {
+    static async reissue(data) {
+        const lock = await CertificateOrderLock.tryLock(data.domain);
+        if (!lock) {
+            throw new TooManyRequestsException("A certificate for this domain is being ordered right now. Please try again in a few minutes.");
+        }
+        try {
+            const claimedRowCount = await data.claimCooldown();
+            if (claimedRowCount === 0) {
+                if (data.lastReissueRequestedAt) {
+                    throw new TooManyRequestsException(CertificateReissueUtil.getCooldownMessage(data.lastReissueRequestedAt, data.now));
+                }
+                /*
+                 * The row matched the cooldown a moment ago and does not now:
+                 * another request claimed it in between, or the domain was deleted.
+                 * Either way the caller must not order.
+                 */
+                throw new BadDataException("Could not start a certificate reissue for this domain. Please refresh the page and try again.");
+            }
+            const outcome = await data.order(lock);
+            if (outcome === CertificateOrderOutcome.Ordered) {
+                return;
+            }
+            // Nothing was ordered, so the customer's reissue was not spent.
+            try {
+                await data.giveCooldownBack();
+            }
+            catch (err) {
+                logger.error(err, { fullDomain: data.domain });
+            }
+            throw new TooManyRequestsException("This installation has used up its new certificates from Let's Encrypt for the moment. Nothing was ordered, and this does not count as your reissue. Please try again in 15 minutes.");
+        }
+        finally {
+            await CertificateOrderLock.release(lock);
+        }
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], CertificateReissueOrder, "reissue", null);
+//# sourceMappingURL=CertificateReissueOrder.js.map

package/build/dist/Server/Utils/Greenlock/CertificateReissueOrder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CertificateReissueOrder.js","sourceRoot":"","sources":["../../../../../Server/Utils/Greenlock/CertificateReissueOrder.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,oBAEN,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,MAAyB,MAAM,WAAW,CAAC;AAClD,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,sBAAsB,MAAM,mCAAmC,CAAC;AACvE,OAAO,gBAAgB,MAAM,2CAA2C,CAAC;AACzE,OAAO,wBAAwB,MAAM,mDAAmD,CAAC;AAEzF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,OAAO,OAAO,uBAAuB;IAEtB,AAAb,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAa3B;QACC,MAAM,IAAI,GACR,MAAM,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAElD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,wBAAwB,CAChC,8FAA8F,CAC/F,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,eAAe,GAAW,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAE3D,IAAI,eAAe,KAAK,CAAC,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;oBAChC,MAAM,IAAI,wBAAwB,CAChC,sBAAsB,CAAC,kBAAkB,CACvC,IAAI,CAAC,sBAAsB,EAC3B,IAAI,CAAC,GAAG,CACT,CACF,CAAC;gBACJ,CAAC;gBAED;;;;mBAIG;gBACH,MAAM,IAAI,gBAAgB,CACxB,+FAA+F,CAChG,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA4B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEhE,IAAI,OAAO,KAAK,uBAAuB,CAAC,OAAO,EAAE,CAAC;gBAChD,OAAO;YACT,CAAC;YAED,gEAAgE;YAChE,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAChC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAmB,CAAC,CAAC;YAClE,CAAC;YAED,MAAM,IAAI,wBAAwB,CAChC,qLAAqL,CACtL,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;CACF;AAlEqB;IADnB,WAAW,EAAE;;;;4CAkEb"}