@oneuptime/common 14.0.13 → 14.0.14

package/build/dist/Server/Utils/Greenlock/CertificateOrder.js

@@ -0,0 +1,353 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import GlobalCache from "../../Infrastructure/GlobalCache";
+import logger from "../Logger";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+import GreenlockUtil from "./Greenlock";
+import CertificateOrderLock from "./CertificateOrderLock";
+import CertificateOrderFailures from "./CertificateOrderFailures";
+import { CertificateOrderOutcome } from "./CertificateOrderOutcome";
+import OneUptimeDate from "../../../Types/Date";
+import Exception from "../../../Types/Exception/Exception";
+import TooManyRequestsException from "../../../Types/Exception/TooManyRequestsException";
+export { CertificateOrderOutcome } from "./CertificateOrderOutcome";
+/*
+ * The first certificate of a custom domain, ordered once.
+ *
+ * A status page domain's certificate can be ordered from three places: the
+ * domain's own "Check now" (the verify-cname route orders the moment the
+ * record is found), the 15-minute sweep that orders for verified domains,
+ * and the sweep that re-orders a certificate that has gone missing. They run
+ * on different replicas, and the two sweeps start on the same tick. Without
+ * this, a domain verified a moment before the sweep starts is ordered twice:
+ * two orders against the one Let's Encrypt account the whole installation
+ * shares, for a name Let's Encrypt allows five certificates a week, and two
+ * http-01 challenges for one name, which clean up each other's challenge
+ * rows.
+ *
+ * So every first order goes through orderIfMissing:
+ *   - one order per name at a time, across every replica: the name's order
+ *     lock (CertificateOrderLock), taken without waiting. A caller that finds
+ *     it taken orders nothing - the order already running is the one it
+ *     would have placed;
+ *   - with the lock held, the certificate table is read again. A name that
+ *     has a certificate - ordered by whoever held the lock a moment ago, or
+ *     a domain that was briefly unverified and is verified again - is only
+ *     recorded as ordered. Ordering it again would spend an order on a
+ *     duplicate, and a renewal that is due is the renewal run's to order.
+ *
+ * Renewals and reissues are not first orders and do not come through here,
+ * but they take the same lock: GreenlockUtil.orderCert places every order
+ * under it.
+ */
+class CertificateOrder {
+    /*
+     * Claims this window's on-demand order for the domain. Exactly one caller
+     * per window gets mayOrder (Redis SET NX); any other gets the last
+     * order's error, if it failed. Without Redis nothing is ordered on
+     * demand: the sweeps order the certificate.
+     */
+    static async claimOnDemandOrder(domain) {
+        const key = CertificateOrder.normalizeDomain(domain);
+        try {
+            const claimed = await GlobalCache.setStringIfNotExists(CertificateOrder.ON_DEMAND_ORDER_NAMESPACE, key, CertificateOrder.ON_DEMAND_ORDERING, {
+                expiresInSeconds: CertificateOrder.ON_DEMAND_ORDER_WINDOW_IN_MINUTES * 60,
+            });
+            if (claimed) {
+                return { mayOrder: true };
+            }
+            const last = await GlobalCache.getString(CertificateOrder.ON_DEMAND_ORDER_NAMESPACE, key);
+            if (last && last !== CertificateOrder.ON_DEMAND_ORDERING) {
+                return { mayOrder: false, lastError: last };
+            }
+            return { mayOrder: false };
+        }
+        catch (err) {
+            logger.error(`Could not check the on-demand certificate order window of ${key}: not ordering now`, { fullDomain: key });
+            logger.error(err, { fullDomain: key });
+            return { mayOrder: false };
+        }
+    }
+    /*
+     * Gives the window's claim back: the claim was taken, but nothing was
+     * ordered - another order of the name was running, or the budget was used
+     * up - so the next click may try again rather than wait out a window for
+     * an order that never happened. Only a claim still marked as ordering is
+     * removed, never a recorded failure. Never throws.
+     */
+    static async releaseOnDemandOrder(domain) {
+        try {
+            await GlobalCache.deleteKeyIfValue(CertificateOrder.ON_DEMAND_ORDER_NAMESPACE, CertificateOrder.normalizeDomain(domain), CertificateOrder.ON_DEMAND_ORDERING);
+        }
+        catch (err) {
+            logger.error(err, { fullDomain: domain });
+        }
+    }
+    /*
+     * Remembers why the on-demand order of this window failed, so a click
+     * within the window shows it instead of a bare "issuing". The key is
+     * written with a whole window again, so the next on-demand order waits a
+     * window from the failure rather than from the click.
+     */
+    static async recordOnDemandOrderFailure(domain, error) {
+        try {
+            await GlobalCache.setString(CertificateOrder.ON_DEMAND_ORDER_NAMESPACE, CertificateOrder.normalizeDomain(domain), error || "We could not order an SSL certificate for this domain.", {
+                expiresInSeconds: CertificateOrder.ON_DEMAND_ORDER_WINDOW_IN_MINUTES * 60,
+            });
+        }
+        catch (err) {
+            logger.error(err, { fullDomain: domain });
+        }
+    }
+    /*
+     * Takes one order from this window's budget: true while fewer than
+     * maxPerWindow were taken in it. A taken order is not given back - an
+     * order that fails still cost the account a validation. Without Redis
+     * nothing is taken, and nothing is ordered.
+     */
+    static async takeOrderSlot(data) {
+        const windowInMs = OneUptimeDate.convertMinutesToMilliseconds(CertificateOrder.ORDER_BUDGET_WINDOW_IN_MINUTES);
+        const windowIndex = Math.floor(data.now.getTime() / windowInMs);
+        try {
+            const taken = await GlobalCache.incrementWithExpiry(CertificateOrder.ORDER_BUDGET_NAMESPACE, `${data.budget}-${windowIndex}`, {
+                // Two windows: long enough for every replica's clock.
+                expiresInSeconds: CertificateOrder.ORDER_BUDGET_WINDOW_IN_MINUTES * 2 * 60,
+            });
+            return taken <= data.maxPerWindow;
+        }
+        catch (err) {
+            logger.error(`Could not read the certificate order budget ${data.budget}: not ordering now`);
+            logger.error(err);
+            return false;
+        }
+    }
+    // One order from a project's on-demand budget of this window.
+    static async takeOnDemandOrderSlot(data) {
+        return await CertificateOrder.takeOrderSlot({
+            budget: `${CertificateOrder.ON_DEMAND_ORDER_BUDGET}-${data.projectId || "unknown-project"}`,
+            maxPerWindow: CertificateOrder.ON_DEMAND_ORDERS_PER_PROJECT_PER_WINDOW,
+            now: data.now,
+        });
+    }
+    /*
+     * The share of orders an order draws from besides the installation's
+     * budget: a sweep's shared budget, a project's on-demand one, or none (a
+     * reissue has a cooldown of its own). Asked by GreenlockUtil.orderCert
+     * once the CNAME check has passed.
+     */
+    static getOrderShare(data) {
+        if (data.sweep) {
+            const sweep = data.sweep;
+            return async () => {
+                return await CertificateOrder.takeOrderSlot({
+                    budget: sweep.budget,
+                    maxPerWindow: sweep.maxPerWindow,
+                    now: OneUptimeDate.getCurrentDate(),
+                });
+            };
+        }
+        if (data.onDemand) {
+            const projectId = data.onDemand.projectId;
+            return async () => {
+                return await CertificateOrder.takeOnDemandOrderSlot({
+                    now: OneUptimeDate.getCurrentDate(),
+                    projectId: projectId,
+                });
+            };
+        }
+        return undefined;
+    }
+    /*
+     * An order somebody asked for through an API (order-ssl), for status page
+     * and dashboard domains alike: at most one per domain per window, shared
+     * with Check now. Answers with what the order did. Throws a 429 for a
+     * click inside the window (with the last order's error, if it failed), for
+     * an order of the name already running and for a used-up budget - after
+     * giving the window back, as nothing was ordered - and rethrows a failed
+     * order once it is remembered for the window.
+     */
+    static async orderOnDemand(data) {
+        const claim = await CertificateOrder.claimOnDemandOrder(data.domain);
+        if (!claim.mayOrder) {
+            throw new TooManyRequestsException(claim.lastError
+                ? `A certificate was ordered for this domain less than ${CertificateOrder.ON_DEMAND_ORDER_WINDOW_IN_MINUTES} minutes ago, and the order failed: ${claim.lastError} Please try again later.`
+                : `A certificate was ordered for this domain less than ${CertificateOrder.ON_DEMAND_ORDER_WINDOW_IN_MINUTES} minutes ago. Please try again later.`);
+        }
+        let outcome;
+        try {
+            outcome = await data.order();
+        }
+        catch (err) {
+            await CertificateOrder.recordOnDemandOrderFailure(data.domain, err instanceof Exception && err.message
+                ? err.message
+                : "We could not order an SSL certificate for this domain.");
+            throw err;
+        }
+        if (outcome === CertificateOrderOutcome.NotOrderedNow ||
+            outcome === CertificateOrderOutcome.LimitReached) {
+            await CertificateOrder.releaseOnDemandOrder(data.domain);
+            throw new TooManyRequestsException(outcome === CertificateOrderOutcome.NotOrderedNow
+                ? "A certificate for this domain is being ordered right now. Please try again in a few minutes."
+                : "This installation has used up its new certificates from Let's Encrypt for the moment. Nothing was ordered: the certificate is ordered automatically shortly, or you can try again in 15 minutes.");
+        }
+        return outcome;
+    }
+    static normalizeDomain(domain) {
+        return CertificateOrderLock.normalizeDomain(domain);
+    }
+    static async orderIfMissing(data) {
+        const domain = CertificateOrder.normalizeDomain(data.domain);
+        const logAttributes = {
+            fullDomain: domain,
+        };
+        const lock = await CertificateOrderLock.tryLock(domain);
+        if (!lock) {
+            return CertificateOrderOutcome.NotOrderedNow;
+        }
+        try {
+            const certificate = (await GreenlockUtil.findCertificatesByDomain([domain])).get(domain);
+            const isUsable = Boolean(certificate &&
+                (!data.renewIfExpired ||
+                    OneUptimeDate.isAfter(certificate.expiresAt, OneUptimeDate.getCurrentDate())));
+            if (isUsable) {
+                logger.debug(`${domain} already has a certificate: recording it as ordered instead of ordering another`, logAttributes);
+                await data.recordAsOrdered();
+                return CertificateOrderOutcome.AlreadyIssued;
+            }
+            return (await data.order(lock)) || CertificateOrderOutcome.Ordered;
+        }
+        finally {
+            await CertificateOrderLock.release(lock);
+        }
+    }
+    /*
+     * Where the certificates of these custom domains stand, by normalized
+     * name: when each one's certificate expires, and why its last order
+     * failed if no order since has succeeded. Two lookups for all of them -
+     * the certificate table and Redis - for a domain list's Status column.
+     */
+    static async getCertificateStates(domains) {
+        const names = Array.from(new Set(domains
+            .map((domain) => {
+            return CertificateOrder.normalizeDomain(domain || "");
+        })
+            .filter((name) => {
+            return name.length > 0;
+        })));
+        const states = new Map();
+        if (names.length === 0) {
+            return states;
+        }
+        const certificates = await GreenlockUtil.findCertificatesByDomain(names);
+        const failures = await CertificateOrderFailures.get(names);
+        for (const name of names) {
+            const certificate = certificates.get(name);
+            const failure = failures.get(name);
+            states.set(name, {
+                certificateExpiresAt: certificate === null || certificate === void 0 ? void 0 : certificate.expiresAt,
+                lastOrderError: failure === null || failure === void 0 ? void 0 : failure.error,
+                lastOrderFailedAt: failure === null || failure === void 0 ? void 0 : failure.failedAt,
+            });
+        }
+        return states;
+    }
+}
+CertificateOrder.LOCK_NAMESPACE = CertificateOrderLock.NAMESPACE;
+CertificateOrder.LOCK_TIMEOUT_IN_MS = CertificateOrderLock.TIMEOUT_IN_MS;
+/*
+ * How often a person may make Check now - or the order API - place an
+ * order for one domain.
+ *
+ * Check now is open to everyone who can read the domain, and an order
+ * that fails leaves the domain unordered, so without this every click
+ * after a failure - or a script - would place another order against the
+ * one Let's Encrypt account the installation shares: 300 new orders per
+ * three hours, and five failed validations per name per hour. Within the
+ * window Check now only reports how the last order went; the sweeps keep
+ * retrying on their own capped schedule.
+ */
+CertificateOrder.ON_DEMAND_ORDER_WINDOW_IN_MINUTES = 15;
+CertificateOrder.ON_DEMAND_ORDER_NAMESPACE = "CustomDomainOnDemandCertificateOrder";
+CertificateOrder.ON_DEMAND_ORDERING = "ordering";
+/*
+ * How many orders one project may place on demand - Check now and the
+ * order API, across all of its domains - in one 15-minute window. Each
+ * domain has its own window above; this bounds a project's sum, so
+ * somebody clicking through many domains whose orders fail cannot use up
+ * the new certificates of the whole installation (CertificateOrderBudget)
+ * and hold up everybody else's. Per project, so one project using its
+ * share up leaves every other project its own.
+ */
+CertificateOrder.ON_DEMAND_ORDER_BUDGET = "CustomDomainOnDemandOrders";
+CertificateOrder.ON_DEMAND_ORDERS_PER_PROJECT_PER_WINDOW = 5;
+/*
+ * A budget of orders that several callers share, per 15-minute window -
+ * the window the sweeps run on and GreenlockUtil.pickForThisRun picks in.
+ *
+ * Each sweep caps its own batch, but several sweeps run on every tick. A
+ * budget they draw from together bounds what they order in a window
+ * between them, however many of them have a backlog: the status page
+ * sweeps share one, the dashboard sweeps another, the on-demand orders a
+ * third. All of them, and the renewals, draw from the installation's
+ * Let's Encrypt allowance as well (CertificateOrderBudget), which
+ * GreenlockUtil.orderCert takes for every order.
+ */
+CertificateOrder.ORDER_BUDGET_NAMESPACE = "CustomDomainCertificateOrderBudget";
+CertificateOrder.ORDER_BUDGET_WINDOW_IN_MINUTES = 15;
+export default CertificateOrder;
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String]),
+    __metadata("design:returntype", Promise)
+], CertificateOrder, "claimOnDemandOrder", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String]),
+    __metadata("design:returntype", Promise)
+], CertificateOrder, "releaseOnDemandOrder", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, String]),
+    __metadata("design:returntype", Promise)
+], CertificateOrder, "recordOnDemandOrderFailure", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], CertificateOrder, "takeOrderSlot", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], CertificateOrder, "takeOnDemandOrderSlot", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], CertificateOrder, "orderOnDemand", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], CertificateOrder, "orderIfMissing", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array]),
+    __metadata("design:returntype", Promise)
+], CertificateOrder, "getCertificateStates", null);
+//# sourceMappingURL=CertificateOrder.js.map

package/build/dist/Server/Utils/Greenlock/CertificateOrder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CertificateOrder.js","sourceRoot":"","sources":["../../../../../Server/Utils/Greenlock/CertificateOrder.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAC3D,OAAO,MAAyB,MAAM,WAAW,CAAC;AAClD,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,aAAa,MAAM,aAAa,CAAC;AACxC,OAAO,oBAEN,MAAM,wBAAwB,CAAC;AAChC,OAAO,wBAEN,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAChD,OAAO,SAAS,MAAM,oCAAoC,CAAC;AAC3D,OAAO,wBAAwB,MAAM,mDAAmD,CAAC;AAEzF,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAqBpE;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAqB,gBAAgB;IAwCnC;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,kBAAkB,CACpC,MAAc;QAEd,MAAM,GAAG,GAAW,gBAAgB,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAE7D,IAAI,CAAC;YACH,MAAM,OAAO,GAAY,MAAM,WAAW,CAAC,oBAAoB,CAC7D,gBAAgB,CAAC,yBAAyB,EAC1C,GAAG,EACH,gBAAgB,CAAC,kBAAkB,EACnC;gBACE,gBAAgB,EACd,gBAAgB,CAAC,iCAAiC,GAAG,EAAE;aAC1D,CACF,CAAC;YAEF,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YAC5B,CAAC;YAED,MAAM,IAAI,GAAkB,MAAM,WAAW,CAAC,SAAS,CACrD,gBAAgB,CAAC,yBAAyB,EAC1C,GAAG,CACJ,CAAC;YAEF,IAAI,IAAI,IAAI,IAAI,KAAK,gBAAgB,CAAC,kBAAkB,EAAE,CAAC;gBACzD,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;YAC9C,CAAC;YAED,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CACV,6DAA6D,GAAG,oBAAoB,EACpF,EAAE,UAAU,EAAE,GAAG,EAAmB,CACrC,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,GAAG,EAAmB,CAAC,CAAC;YAExD,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,MAAc;QACrD,IAAI,CAAC;YACH,MAAM,WAAW,CAAC,gBAAgB,CAChC,gBAAgB,CAAC,yBAAyB,EAC1C,gBAAgB,CAAC,eAAe,CAAC,MAAM,CAAC,EACxC,gBAAgB,CAAC,kBAAkB,CACpC,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,EAAmB,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAC5C,MAAc,EACd,KAAa;QAEb,IAAI,CAAC;YACH,MAAM,WAAW,CAAC,SAAS,CACzB,gBAAgB,CAAC,yBAAyB,EAC1C,gBAAgB,CAAC,eAAe,CAAC,MAAM,CAAC,EACxC,KAAK,IAAI,wDAAwD,EACjE;gBACE,gBAAgB,EACd,gBAAgB,CAAC,iCAAiC,GAAG,EAAE;aAC1D,CACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,EAAmB,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAmBD;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAIjC;QACC,MAAM,UAAU,GAAW,aAAa,CAAC,4BAA4B,CACnE,gBAAgB,CAAC,8BAA8B,CAChD,CAAC;QAEF,MAAM,WAAW,GAAW,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,UAAU,CAAC,CAAC;QAExE,IAAI,CAAC;YACH,MAAM,KAAK,GAAW,MAAM,WAAW,CAAC,mBAAmB,CACzD,gBAAgB,CAAC,sBAAsB,EACvC,GAAG,IAAI,CAAC,MAAM,IAAI,WAAW,EAAE,EAC/B;gBACE,sDAAsD;gBACtD,gBAAgB,EACd,gBAAgB,CAAC,8BAA8B,GAAG,CAAC,GAAG,EAAE;aAC3D,CACF,CAAC;YAEF,OAAO,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC;QACpC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CACV,+CAA+C,IAAI,CAAC,MAAM,oBAAoB,CAC/E,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAElB,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,8DAA8D;IAE1C,AAAb,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,IAGzC;QACC,OAAO,MAAM,gBAAgB,CAAC,aAAa,CAAC;YAC1C,MAAM,EAAE,GAAG,gBAAgB,CAAC,sBAAsB,IAAI,IAAI,CAAC,SAAS,IAAI,iBAAiB,EAAE;YAC3F,YAAY,EAAE,gBAAgB,CAAC,uCAAuC;YACtE,GAAG,EAAE,IAAI,CAAC,GAAG;SACd,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,aAAa,CAAC,IAG3B;QACC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,GAA6C,IAAI,CAAC,KAAK,CAAC;YAEnE,OAAO,KAAK,IAAsB,EAAE;gBAClC,OAAO,MAAM,gBAAgB,CAAC,aAAa,CAAC;oBAC1C,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,GAAG,EAAE,aAAa,CAAC,cAAc,EAAE;iBACpC,CAAC,CAAC;YACL,CAAC,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,SAAS,GAAuB,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;YAE9D,OAAO,KAAK,IAAsB,EAAE;gBAClC,OAAO,MAAM,gBAAgB,CAAC,qBAAqB,CAAC;oBAClD,GAAG,EAAE,aAAa,CAAC,cAAc,EAAE;oBACnC,SAAS,EAAE,SAAS;iBACrB,CAAC,CAAC;YACL,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;;OAQG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAGjC;QACC,MAAM,KAAK,GAAuB,MAAM,gBAAgB,CAAC,kBAAkB,CACzE,IAAI,CAAC,MAAM,CACZ,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpB,MAAM,IAAI,wBAAwB,CAChC,KAAK,CAAC,SAAS;gBACb,CAAC,CAAC,uDAAuD,gBAAgB,CAAC,iCAAiC,uCAAuC,KAAK,CAAC,SAAS,0BAA0B;gBAC3L,CAAC,CAAC,uDAAuD,gBAAgB,CAAC,iCAAiC,uCAAuC,CACrJ,CAAC;QACJ,CAAC;QAED,IAAI,OAAgC,CAAC;QAErC,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,gBAAgB,CAAC,0BAA0B,CAC/C,IAAI,CAAC,MAAM,EACX,GAAG,YAAY,SAAS,IAAI,GAAG,CAAC,OAAO;gBACrC,CAAC,CAAC,GAAG,CAAC,OAAO;gBACb,CAAC,CAAC,wDAAwD,CAC7D,CAAC;YAEF,MAAM,GAAG,CAAC;QACZ,CAAC;QAED,IACE,OAAO,KAAK,uBAAuB,CAAC,aAAa;YACjD,OAAO,KAAK,uBAAuB,CAAC,YAAY,EAChD,CAAC;YACD,MAAM,gBAAgB,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEzD,MAAM,IAAI,wBAAwB,CAChC,OAAO,KAAK,uBAAuB,CAAC,aAAa;gBAC/C,CAAC,CAAC,8FAA8F;gBAChG,CAAC,CAAC,kMAAkM,CACvM,CAAC;QACJ,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEM,MAAM,CAAC,eAAe,CAAC,MAAc;QAC1C,OAAO,oBAAoB,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IACtD,CAAC;IAGmB,AAAb,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,IAkBlC;QACC,MAAM,MAAM,GAAW,gBAAgB,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAErE,MAAM,aAAa,GAAkB;YACnC,UAAU,EAAE,MAAM;SACF,CAAC;QAEnB,MAAM,IAAI,GACR,MAAM,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAE7C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,uBAAuB,CAAC,aAAa,CAAC;QAC/C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,WAAW,GAAgC,CAC/C,MAAM,aAAa,CAAC,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC,CACvD,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAEd,MAAM,QAAQ,GAAY,OAAO,CAC/B,WAAW;gBACT,CAAC,CAAC,IAAI,CAAC,cAAc;oBACnB,aAAa,CAAC,OAAO,CACnB,WAAW,CAAC,SAAiB,EAC7B,aAAa,CAAC,cAAc,EAAE,CAC/B,CAAC,CACP,CAAC;YAEF,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CACV,GAAG,MAAM,iFAAiF,EAC1F,aAAa,CACd,CAAC;gBAEF,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;gBAE7B,OAAO,uBAAuB,CAAC,aAAa,CAAC;YAC/C,CAAC;YAED,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,uBAAuB,CAAC,OAAO,CAAC;QACrE,CAAC;gBAAS,CAAC;YACT,MAAM,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,oBAAoB,CACtC,OAAsB;QAEtB,MAAM,KAAK,GAAkB,KAAK,CAAC,IAAI,CACrC,IAAI,GAAG,CACL,OAAO;aACJ,GAAG,CAAC,CAAC,MAAc,EAAE,EAAE;YACtB,OAAO,gBAAgB,CAAC,eAAe,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAY,EAAE,EAAE;YACvB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACzB,CAAC,CAAC,CACL,CACF,CAAC;QAEF,MAAM,MAAM,GAA8C,IAAI,GAAG,EAG9D,CAAC;QAEJ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,YAAY,GAChB,MAAM,aAAa,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,QAAQ,GACZ,MAAM,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAE5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,WAAW,GAAgC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACxE,MAAM,OAAO,GAAwC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAExE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE;gBACf,oBAAoB,EAAE,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,SAAS;gBAC5C,cAAc,EAAE,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK;gBAC9B,iBAAiB,EAAE,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,QAAQ;aACrC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;;AA7ZsB,+BAAc,GACnC,oBAAoB,CAAC,SAAS,CAAC;AAEV,mCAAkB,GACvC,oBAAoB,CAAC,aAAa,CAAC;AAErC;;;;;;;;;;;GAWG;AACoB,kDAAiC,GAAW,EAAE,CAAC;AAE/C,0CAAyB,GAC9C,sCAAsC,CAAC;AAEjB,mCAAkB,GAAW,UAAU,CAAC;AAEhE;;;;;;;;GAQG;AACoB,uCAAsB,GAC3C,4BAA4B,CAAC;AAER,wDAAuC,GAAW,CAAC,CAAC;AAgG3E;;;;;;;;;;;GAWG;AACoB,uCAAsB,GAC3C,oCAAoC,CAAC;AAEhB,+CAA8B,GAAW,EAAE,CAAC;eArJhD,gBAAgB;AA+Cf;IADnB,WAAW,EAAE;;;;gDAwCb;AAUmB;IADnB,WAAW,EAAE;;;;kDAWb;AASmB;IADnB,WAAW,EAAE;;;;wDAkBb;AA0BmB;IADnB,WAAW,EAAE;;;;2CAgCb;AAImB;IADnB,WAAW,EAAE;;;;mDAUb;AAgDmB;IADnB,WAAW,EAAE;;;;2CA8Cb;AAOmB;IADnB,WAAW,EAAE;;;;4CA8Db;AASmB;IADnB,WAAW,EAAE;;qCAEH,KAAK;;kDAyCf"}

package/build/dist/Server/Utils/Greenlock/CertificateOrderBudget.js

@@ -0,0 +1,119 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import GlobalCache from "../../Infrastructure/GlobalCache";
+import logger from "../Logger";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+import OneUptimeDate from "../../../Types/Date";
+// Why a certificate is ordered, which decides what the order may spend.
+export var CertificateOrderReason;
+(function (CertificateOrderReason) {
+    // A certificate that is serving comes due for renewal, or has expired.
+    CertificateOrderReason["Renewal"] = "Renewal";
+    // The installation's own host, ordered and renewed by CoreSSL.
+    CertificateOrderReason["PrimaryHost"] = "PrimaryHost";
+    // A custom domain's first certificate, or one that has gone missing.
+    CertificateOrderReason["FirstCertificate"] = "FirstCertificate";
+    // A brand new certificate somebody asked for (Reissue SSL).
+    CertificateOrderReason["Reissue"] = "Reissue";
+})(CertificateOrderReason || (CertificateOrderReason = {}));
+/*
+ * The Let's Encrypt allowance of the whole installation, shared by every
+ * order: status page and dashboard domains, first orders, re-orders,
+ * reissues, Check now, the order APIs, the renewal runs and the primary host.
+ *
+ * Every certificate this installation orders comes from one Let's Encrypt
+ * account (LETSENCRYPT_ACCOUNT_KEY), and Let's Encrypt allows an account
+ * 300 new orders per three hours, renewals included. Each sweep and run has
+ * a cap of its own, but those caps add up: with backlogs, the status page
+ * and dashboard sweeps and renewals could ask for more than 400 orders in
+ * three hours, and once the account is refused, everybody's renewals are
+ * refused with it. So GreenlockUtil.orderCert takes one unit from this
+ * budget for every order it places, whoever asked, and orders nothing when
+ * there is none left. A window is 15 minutes, the schedule the sweeps run
+ * on; the next window has a new budget.
+ *
+ * Renewals come first. They keep a domain that is served today served
+ * tomorrow, so they may use the whole window, while new certificates - first
+ * orders, re-orders of a missing one, reissues - may only use part of it: a
+ * flood of new domains, or someone clicking Check now on many of them, can
+ * never leave the renewals without orders. The primary host counts as a
+ * renewal: it is the installation itself.
+ *
+ * One counter holds both limits (GlobalCache.incrementIfBelow), and a
+ * refused request adds nothing to it, so new certificates refused at their
+ * limit do not use up what renewals still may take.
+ */
+class CertificateOrderBudget {
+    static getLimit(reason) {
+        switch (reason) {
+            case CertificateOrderReason.Renewal:
+            case CertificateOrderReason.PrimaryHost:
+                return CertificateOrderBudget.ORDERS_PER_WINDOW;
+            case CertificateOrderReason.FirstCertificate:
+            case CertificateOrderReason.Reissue:
+            default:
+                return CertificateOrderBudget.NEW_CERTIFICATE_ORDERS_PER_WINDOW;
+        }
+    }
+    // The 15-minute window a moment falls in; the same on every replica.
+    static getWindowIndex(now) {
+        return Math.floor(now.getTime() /
+            OneUptimeDate.convertMinutesToMilliseconds(CertificateOrderBudget.WINDOW_IN_MINUTES));
+    }
+    /*
+     * Takes one order from this window's budget: true when the order may be
+     * placed. A taken order is not given back, even if the order fails - a
+     * failed order still counted against the account. Without Redis nothing
+     * is taken, and nothing is ordered.
+     */
+    static async takeSlot(data) {
+        const limit = CertificateOrderBudget.getLimit(data.reason);
+        try {
+            const taken = await GlobalCache.incrementIfBelow(CertificateOrderBudget.NAMESPACE, `window-${CertificateOrderBudget.getWindowIndex(data.now)}`, {
+                limit: limit,
+                // Two windows: long enough for every replica's clock.
+                expiresInSeconds: CertificateOrderBudget.WINDOW_IN_MINUTES * 2 * 60,
+            });
+            if (taken === null) {
+                logger.warn(`Not ordering a certificate${data.domain ? ` for ${data.domain}` : ""} now: this installation's Let's Encrypt orders for this 15-minute window are used up (${limit} for a ${data.reason} order). It is tried again in a later window.`, { fullDomain: data.domain });
+                return false;
+            }
+            return true;
+        }
+        catch (err) {
+            logger.error("Could not read this installation's Let's Encrypt order budget: not ordering now", { fullDomain: data.domain });
+            logger.error(err, { fullDomain: data.domain });
+            return false;
+        }
+    }
+}
+CertificateOrderBudget.NAMESPACE = "LetsEncryptAccountOrderBudget";
+CertificateOrderBudget.WINDOW_IN_MINUTES = 15;
+// Let's Encrypt's limit for one account, renewals included.
+CertificateOrderBudget.LETS_ENCRYPT_ORDERS_PER_THREE_HOURS = 300;
+/*
+ * Orders of any kind in one window. Any three hours overlap at most 13
+ * windows, so at most 260 orders: under Let's Encrypt's 300, with room for
+ * the orders a clock that is a little off puts in a neighbouring window.
+ */
+CertificateOrderBudget.ORDERS_PER_WINDOW = 20;
+/*
+ * Of those, how many may be new certificates. Whatever they take, 8 orders
+ * per window stay for renewals and the primary host - more than 750 a day.
+ */
+CertificateOrderBudget.NEW_CERTIFICATE_ORDERS_PER_WINDOW = 12;
+export default CertificateOrderBudget;
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], CertificateOrderBudget, "takeSlot", null);
+//# sourceMappingURL=CertificateOrderBudget.js.map

package/build/dist/Server/Utils/Greenlock/CertificateOrderBudget.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CertificateOrderBudget.js","sourceRoot":"","sources":["../../../../../Server/Utils/Greenlock/CertificateOrderBudget.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAC3D,OAAO,MAAyB,MAAM,WAAW,CAAC;AAClD,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,aAAa,MAAM,qBAAqB,CAAC;AAEhD,wEAAwE;AACxE,MAAM,CAAN,IAAY,sBASX;AATD,WAAY,sBAAsB;IAChC,uEAAuE;IACvE,6CAAmB,CAAA;IACnB,+DAA+D;IAC/D,qDAA2B,CAAA;IAC3B,qEAAqE;IACrE,+DAAqC,CAAA;IACrC,4DAA4D;IAC5D,6CAAmB,CAAA;AACrB,CAAC,EATW,sBAAsB,KAAtB,sBAAsB,QASjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAqB,sBAAsB;IAqBlC,MAAM,CAAC,QAAQ,CAAC,MAA8B;QACnD,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,sBAAsB,CAAC,OAAO,CAAC;YACpC,KAAK,sBAAsB,CAAC,WAAW;gBACrC,OAAO,sBAAsB,CAAC,iBAAiB,CAAC;YAClD,KAAK,sBAAsB,CAAC,gBAAgB,CAAC;YAC7C,KAAK,sBAAsB,CAAC,OAAO,CAAC;YACpC;gBACE,OAAO,sBAAsB,CAAC,iCAAiC,CAAC;QACpE,CAAC;IACH,CAAC;IAED,qEAAqE;IAC9D,MAAM,CAAC,cAAc,CAAC,GAAS;QACpC,OAAO,IAAI,CAAC,KAAK,CACf,GAAG,CAAC,OAAO,EAAE;YACX,aAAa,CAAC,4BAA4B,CACxC,sBAAsB,CAAC,iBAAiB,CACzC,CACJ,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAK5B;QACC,MAAM,KAAK,GAAW,sBAAsB,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEnE,IAAI,CAAC;YACH,MAAM,KAAK,GAAkB,MAAM,WAAW,CAAC,gBAAgB,CAC7D,sBAAsB,CAAC,SAAS,EAChC,UAAU,sBAAsB,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAC3D;gBACE,KAAK,EAAE,KAAK;gBACZ,sDAAsD;gBACtD,gBAAgB,EAAE,sBAAsB,CAAC,iBAAiB,GAAG,CAAC,GAAG,EAAE;aACpE,CACF,CAAC;YAEF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CACT,6BAA6B,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,yFAAyF,KAAK,UAAU,IAAI,CAAC,MAAM,+CAA+C,EACvO,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAmB,CAC7C,CAAC;gBAEF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CACV,iFAAiF,EACjF,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAmB,CAC7C,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAmB,CAAC,CAAC;YAEhE,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;;AAvFsB,gCAAS,GAAW,+BAA+B,CAAC;AAEpD,wCAAiB,GAAW,EAAE,CAAC;AAEtD,4DAA4D;AACrC,0DAAmC,GAAW,GAAG,CAAC;AAEzE;;;;GAIG;AACoB,wCAAiB,GAAW,EAAE,CAAC;AAEtD;;;GAGG;AACoB,wDAAiC,GAAW,EAAE,CAAC;eAnBnD,sBAAsB;AAkDrB;IADnB,WAAW,EAAE;;;;4CAuCb"}