@oneuptime/common 14.0.13 → 14.0.14

package/Server/Utils/Greenlock/CertificateOrderBudget.ts

@@ -0,0 +1,134 @@
+import GlobalCache from "../../Infrastructure/GlobalCache";
+import logger, { LogAttributes } from "../Logger";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+import OneUptimeDate from "../../../Types/Date";
+
+// Why a certificate is ordered, which decides what the order may spend.
+export enum CertificateOrderReason {
+  // A certificate that is serving comes due for renewal, or has expired.
+  Renewal = "Renewal",
+  // The installation's own host, ordered and renewed by CoreSSL.
+  PrimaryHost = "PrimaryHost",
+  // A custom domain's first certificate, or one that has gone missing.
+  FirstCertificate = "FirstCertificate",
+  // A brand new certificate somebody asked for (Reissue SSL).
+  Reissue = "Reissue",
+}
+
+/*
+ * The Let's Encrypt allowance of the whole installation, shared by every
+ * order: status page and dashboard domains, first orders, re-orders,
+ * reissues, Check now, the order APIs, the renewal runs and the primary host.
+ *
+ * Every certificate this installation orders comes from one Let's Encrypt
+ * account (LETSENCRYPT_ACCOUNT_KEY), and Let's Encrypt allows an account
+ * 300 new orders per three hours, renewals included. Each sweep and run has
+ * a cap of its own, but those caps add up: with backlogs, the status page
+ * and dashboard sweeps and renewals could ask for more than 400 orders in
+ * three hours, and once the account is refused, everybody's renewals are
+ * refused with it. So GreenlockUtil.orderCert takes one unit from this
+ * budget for every order it places, whoever asked, and orders nothing when
+ * there is none left. A window is 15 minutes, the schedule the sweeps run
+ * on; the next window has a new budget.
+ *
+ * Renewals come first. They keep a domain that is served today served
+ * tomorrow, so they may use the whole window, while new certificates - first
+ * orders, re-orders of a missing one, reissues - may only use part of it: a
+ * flood of new domains, or someone clicking Check now on many of them, can
+ * never leave the renewals without orders. The primary host counts as a
+ * renewal: it is the installation itself.
+ *
+ * One counter holds both limits (GlobalCache.incrementIfBelow), and a
+ * refused request adds nothing to it, so new certificates refused at their
+ * limit do not use up what renewals still may take.
+ */
+export default class CertificateOrderBudget {
+  public static readonly NAMESPACE: string = "LetsEncryptAccountOrderBudget";
+
+  public static readonly WINDOW_IN_MINUTES: number = 15;
+
+  // Let's Encrypt's limit for one account, renewals included.
+  public static readonly LETS_ENCRYPT_ORDERS_PER_THREE_HOURS: number = 300;
+
+  /*
+   * Orders of any kind in one window. Any three hours overlap at most 13
+   * windows, so at most 260 orders: under Let's Encrypt's 300, with room for
+   * the orders a clock that is a little off puts in a neighbouring window.
+   */
+  public static readonly ORDERS_PER_WINDOW: number = 20;
+
+  /*
+   * Of those, how many may be new certificates. Whatever they take, 8 orders
+   * per window stay for renewals and the primary host - more than 750 a day.
+   */
+  public static readonly NEW_CERTIFICATE_ORDERS_PER_WINDOW: number = 12;
+
+  public static getLimit(reason: CertificateOrderReason): number {
+    switch (reason) {
+      case CertificateOrderReason.Renewal:
+      case CertificateOrderReason.PrimaryHost:
+        return CertificateOrderBudget.ORDERS_PER_WINDOW;
+      case CertificateOrderReason.FirstCertificate:
+      case CertificateOrderReason.Reissue:
+      default:
+        return CertificateOrderBudget.NEW_CERTIFICATE_ORDERS_PER_WINDOW;
+    }
+  }
+
+  // The 15-minute window a moment falls in; the same on every replica.
+  public static getWindowIndex(now: Date): number {
+    return Math.floor(
+      now.getTime() /
+        OneUptimeDate.convertMinutesToMilliseconds(
+          CertificateOrderBudget.WINDOW_IN_MINUTES,
+        ),
+    );
+  }
+
+  /*
+   * Takes one order from this window's budget: true when the order may be
+   * placed. A taken order is not given back, even if the order fails - a
+   * failed order still counted against the account. Without Redis nothing
+   * is taken, and nothing is ordered.
+   */
+  @CaptureSpan()
+  public static async takeSlot(data: {
+    reason: CertificateOrderReason;
+    now: Date;
+    // For the log line.
+    domain?: string | undefined;
+  }): Promise<boolean> {
+    const limit: number = CertificateOrderBudget.getLimit(data.reason);
+
+    try {
+      const taken: number | null = await GlobalCache.incrementIfBelow(
+        CertificateOrderBudget.NAMESPACE,
+        `window-${CertificateOrderBudget.getWindowIndex(data.now)}`,
+        {
+          limit: limit,
+          // Two windows: long enough for every replica's clock.
+          expiresInSeconds: CertificateOrderBudget.WINDOW_IN_MINUTES * 2 * 60,
+        },
+      );
+
+      if (taken === null) {
+        logger.warn(
+          `Not ordering a certificate${data.domain ? ` for ${data.domain}` : ""} now: this installation's Let's Encrypt orders for this 15-minute window are used up (${limit} for a ${data.reason} order). It is tried again in a later window.`,
+          { fullDomain: data.domain } as LogAttributes,
+        );
+
+        return false;
+      }
+
+      return true;
+    } catch (err) {
+      logger.error(
+        "Could not read this installation's Let's Encrypt order budget: not ordering now",
+        { fullDomain: data.domain } as LogAttributes,
+      );
+      logger.error(err, { fullDomain: data.domain } as LogAttributes);
+
+      return false;
+    }
+  }
+}

package/Server/Utils/Greenlock/CertificateOrderFailures.ts

@@ -0,0 +1,288 @@
+import GlobalCache from "../../Infrastructure/GlobalCache";
+import CertificateOrderLock from "./CertificateOrderLock";
+import logger, { LogAttributes } from "../Logger";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+import OneUptimeDate from "../../../Types/Date";
+
+export interface CertificateOrderFailure {
+  // What the order threw, as the person who placed it was told.
+  error: string;
+  failedAt: Date;
+  // How many orders for the name have failed in a row.
+  failures: number;
+}
+
+/*
+ * The last failed certificate order of each name, kept in Redis.
+ *
+ * An order that fails leaves nothing behind in the database: the domain
+ * stays unordered, and the sweeps order it again. So a domain whose order
+ * keeps failing - a CAA record that leaves Let's Encrypt out, a server Let's
+ * Encrypt cannot reach - said "Issuing a free certificate" for days, and the
+ * reason was only in the worker's log. GreenlockUtil.orderCert records every
+ * failed order here and clears the record when an order for the name
+ * succeeds, so:
+ *
+ *   - the domain's Status column shows that the order failed, and why;
+ *   - the sweeps and renewal runs, which would otherwise order a failing name
+ *     again every 15 minutes, wait longer after each failure in a row
+ *     (getRetryDelayInMinutes). Every failed order still counts against the
+ *     installation's shared Let's Encrypt account, and Let's Encrypt allows
+ *     five failed validations per name per hour.
+ *
+ * Check now and the order API do not wait: somebody who has just fixed the
+ * cause should not have to. They have a window of their own per domain
+ * (CertificateOrder.claimOnDemandOrder).
+ *
+ * Kept in Redis rather than on the domain row, so neither table changes:
+ * losing a record (a Redis restart) only means the next failure is recorded
+ * again. Every method here never throws.
+ */
+export default class CertificateOrderFailures {
+  public static readonly NAMESPACE: string =
+    "CustomDomainCertificateOrderFailure";
+
+  /*
+   * How long a record is kept without another failure. The sweeps retry a
+   * failing name at least every LONGEST_RETRY_DELAY_IN_HOURS, which renews
+   * it well before this.
+   */
+  public static readonly KEEP_FOR_IN_DAYS: number = 7;
+
+  // How long the automatic attempts wait after one failure...
+  public static readonly FIRST_RETRY_DELAY_IN_MINUTES: number = 15;
+
+  // ...doubling with each further failure in a row, up to this.
+  public static readonly LONGEST_RETRY_DELAY_IN_HOURS: number = 6;
+
+  /*
+   * A failure is recorded when the order ends, which can be a minute or two
+   * after the sweep started it. Without this, the sweep on the tick the
+   * delay ends on would find the name still waiting, and the first retry
+   * would come a tick late.
+   */
+  public static readonly RETRY_GRACE_IN_MINUTES: number = 2;
+
+  private static readonly LOOKUP_CHUNK_SIZE: number = 500;
+
+  public static getRetryDelayInMinutes(failures: number): number {
+    const failuresInARow: number = Math.max(1, Math.floor(failures || 1));
+
+    const longest: number =
+      CertificateOrderFailures.LONGEST_RETRY_DELAY_IN_HOURS * 60;
+
+    // Past this many doublings the delay is the longest one anyway.
+    if (failuresInARow > 20) {
+      return longest;
+    }
+
+    return Math.min(
+      CertificateOrderFailures.FIRST_RETRY_DELAY_IN_MINUTES *
+        Math.pow(2, failuresInARow - 1),
+      longest,
+    );
+  }
+
+  // When the sweeps and renewal runs order this name again.
+  public static getRetryAt(failure: CertificateOrderFailure): Date {
+    return OneUptimeDate.addRemoveMinutes(
+      failure.failedAt,
+      CertificateOrderFailures.getRetryDelayInMinutes(failure.failures),
+    );
+  }
+
+  public static isWaitingToRetry(
+    failure: CertificateOrderFailure | undefined,
+    now: Date,
+  ): boolean {
+    if (!failure) {
+      return false;
+    }
+
+    return OneUptimeDate.isAfter(
+      OneUptimeDate.addRemoveMinutes(
+        CertificateOrderFailures.getRetryAt(failure),
+        -CertificateOrderFailures.RETRY_GRACE_IN_MINUTES,
+      ),
+      now,
+    );
+  }
+
+  @CaptureSpan()
+  public static async record(data: {
+    domain: string;
+    error: string;
+    now: Date;
+  }): Promise<void> {
+    const name: string = CertificateOrderFailures.normalizeDomain(data.domain);
+
+    try {
+      const previous: CertificateOrderFailure | null =
+        CertificateOrderFailures.parse(
+          await GlobalCache.getString(CertificateOrderFailures.NAMESPACE, name),
+        );
+
+      const failure: CertificateOrderFailure = {
+        error:
+          data.error.trim() ||
+          "We could not order an SSL certificate for this domain.",
+        failedAt: data.now,
+        failures: (previous?.failures || 0) + 1,
+      };
+
+      await GlobalCache.setString(
+        CertificateOrderFailures.NAMESPACE,
+        name,
+        JSON.stringify({
+          error: failure.error,
+          failedAt: failure.failedAt.toISOString(),
+          failures: failure.failures,
+        }),
+        {
+          expiresInSeconds: OneUptimeDate.getSecondsInDays(
+            CertificateOrderFailures.KEEP_FOR_IN_DAYS,
+          ),
+        },
+      );
+    } catch (err) {
+      logger.error(`Could not record the failed certificate order of ${name}`, {
+        fullDomain: name,
+      } as LogAttributes);
+      logger.error(err, { fullDomain: name } as LogAttributes);
+    }
+  }
+
+  // An order for the name succeeded: forget its failures.
+  @CaptureSpan()
+  public static async clear(domain: string): Promise<void> {
+    const name: string = CertificateOrderFailures.normalizeDomain(domain);
+
+    try {
+      await GlobalCache.deleteKey(CertificateOrderFailures.NAMESPACE, name);
+    } catch (err) {
+      logger.error(err, { fullDomain: name } as LogAttributes);
+    }
+  }
+
+  /*
+   * The recorded failure of each of these names that has one, by normalized
+   * name, a few hundred names per round trip. Empty when Redis cannot be
+   * read: then nothing waits, and nothing shows a failure.
+   */
+  @CaptureSpan()
+  public static async get(
+    domains: Array<string>,
+  ): Promise<Map<string, CertificateOrderFailure>> {
+    const failures: Map<string, CertificateOrderFailure> = new Map<
+      string,
+      CertificateOrderFailure
+    >();
+
+    const names: Array<string> = Array.from(
+      new Set<string>(
+        domains
+          .map((domain: string) => {
+            return CertificateOrderFailures.normalizeDomain(domain);
+          })
+          .filter((name: string) => {
+            return name.length > 0;
+          }),
+      ),
+    );
+
+    try {
+      for (
+        let offset: number = 0;
+        offset < names.length;
+        offset += CertificateOrderFailures.LOOKUP_CHUNK_SIZE
+      ) {
+        const chunk: Array<string> = names.slice(
+          offset,
+          offset + CertificateOrderFailures.LOOKUP_CHUNK_SIZE,
+        );
+
+        const values: Array<string | null> = await GlobalCache.getStrings(
+          CertificateOrderFailures.NAMESPACE,
+          chunk,
+        );
+
+        chunk.forEach((name: string, index: number) => {
+          const failure: CertificateOrderFailure | null =
+            CertificateOrderFailures.parse(values[index] ?? null);
+
+          if (failure) {
+            failures.set(name, failure);
+          }
+        });
+      }
+    } catch (err) {
+      logger.error("Could not read the failed certificate orders");
+      logger.error(err);
+
+      return new Map<string, CertificateOrderFailure>();
+    }
+
+    return failures;
+  }
+
+  /*
+   * The items whose name is not waiting after a failed order - what a sweep
+   * or a renewal run may order now.
+   */
+  public static async withoutThoseWaitingToRetry<T>(data: {
+    items: Array<T>;
+    getDomain: (item: T) => string;
+    now: Date;
+  }): Promise<Array<T>> {
+    if (data.items.length === 0) {
+      return [];
+    }
+
+    const failures: Map<string, CertificateOrderFailure> =
+      await CertificateOrderFailures.get(data.items.map(data.getDomain));
+
+    return data.items.filter((item: T) => {
+      return !CertificateOrderFailures.isWaitingToRetry(
+        failures.get(
+          CertificateOrderFailures.normalizeDomain(data.getDomain(item)),
+        ),
+        data.now,
+      );
+    });
+  }
+
+  // The same name the order lock and the certificate table use.
+  public static normalizeDomain(domain: string): string {
+    return CertificateOrderLock.normalizeDomain(domain || "");
+  }
+
+  private static parse(value: string | null): CertificateOrderFailure | null {
+    if (!value) {
+      return null;
+    }
+
+    try {
+      const json: {
+        error?: unknown;
+        failedAt?: unknown;
+        failures?: unknown;
+      } = JSON.parse(value);
+
+      const failedAt: Date = new Date(String(json.failedAt));
+
+      if (typeof json.error !== "string" || isNaN(failedAt.getTime())) {
+        return null;
+      }
+
+      const failures: number = Number(json.failures);
+
+      return {
+        error: json.error,
+        failedAt: failedAt,
+        failures: Number.isFinite(failures) && failures > 0 ? failures : 1,
+      };
+    } catch {
+      return null;
+    }
+  }
+}

package/Server/Utils/Greenlock/CertificateOrderLock.ts

@@ -0,0 +1,163 @@
+import Semaphore, {
+  SemaphoreLockTimeoutError,
+  SemaphoreMutex,
+} from "../../Infrastructure/Semaphore";
+import logger, { LogAttributes } from "../Logger";
+import OneUptimeDate from "../../../Types/Date";
+
+/*
+ * Proof that its holder holds the order lock of one name. Only tryLock makes
+ * one, and it stops counting once released.
+ */
+export interface CertificateOrderLockHandle {
+  // The name it locks, normalized.
+  readonly domain: string;
+}
+
+/*
+ * One certificate order per name at a time, across every replica.
+ *
+ * A name's certificate can be ordered from many places: Check now, the
+ * sweeps that order first certificates and re-order missing ones, the order
+ * and reissue APIs, the renewal runs and, for the installation's own host,
+ * CoreSSL. Two orders for one name at once are two orders against the one
+ * Let's Encrypt account the whole installation shares, and two http-01
+ * challenges for one name, whose clean-up deletes the challenge rows by
+ * name - each order can remove the other's challenge while Let's Encrypt is
+ * still checking it.
+ *
+ * GreenlockUtil.orderCert takes this lock for every order, or checks that its
+ * caller holds it (CertificateOrder.orderIfMissing takes it first, to look
+ * the name up again before deciding to order). The lock is taken without
+ * waiting: a caller that finds it taken orders nothing, because the order
+ * already running is the one it would have placed.
+ */
+export default class CertificateOrderLock {
+  public static readonly NAMESPACE: string = "CustomDomainCertificateOrder";
+
+  /*
+   * How long the lock outlives a holder that died without releasing it. A
+   * live holder refreshes it (redis-semaphore does, every 80% of this), so
+   * an order that takes longer - the CA polls a challenge for a while - keeps
+   * it.
+   */
+  public static readonly TIMEOUT_IN_MS: number =
+    OneUptimeDate.convertMinutesToMilliseconds(2);
+
+  // The handles tryLock gave out and that are not released yet.
+  private static readonly held: WeakMap<
+    CertificateOrderLockHandle,
+    SemaphoreMutex
+  > = new WeakMap<CertificateOrderLockHandle, SemaphoreMutex>();
+
+  /*
+   * Handles whose lock Redis lost while they held it - a refresh that
+   * failed, an eviction. Another replica can take the name's lock from then
+   * on, so such a handle holds nothing any more.
+   */
+  private static readonly lost: WeakSet<CertificateOrderLockHandle> =
+    new WeakSet<CertificateOrderLockHandle>();
+
+  public static normalizeDomain(domain: string): string {
+    return domain.trim().toLowerCase();
+  }
+
+  /*
+   * The lock on the name, or null when it is taken - or cannot be taken at
+   * all. Without Redis nothing is ordered: the sweeps run from Redis' own
+   * job queue, and ordering without the lock is what this is here to stop.
+   */
+  public static async tryLock(
+    domain: string,
+  ): Promise<CertificateOrderLockHandle | null> {
+    const name: string = CertificateOrderLock.normalizeDomain(domain);
+
+    const handle: CertificateOrderLockHandle = Object.freeze({
+      domain: name,
+    });
+
+    try {
+      const mutex: SemaphoreMutex = await Semaphore.lock({
+        key: name,
+        namespace: CertificateOrderLock.NAMESPACE,
+        lockTimeout: CertificateOrderLock.TIMEOUT_IN_MS,
+        acquireAttemptsLimit: 1,
+        onLockLost: (err: Error) => {
+          // From now on this handle holds nothing (isHeldFor says so).
+          CertificateOrderLock.held.delete(handle);
+          CertificateOrderLock.lost.add(handle);
+
+          logger.error(
+            `Lost the certificate order lock of ${name} while ordering`,
+            { fullDomain: name } as LogAttributes,
+          );
+          logger.error(err, { fullDomain: name } as LogAttributes);
+        },
+      });
+
+      CertificateOrderLock.held.set(handle, mutex);
+
+      return handle;
+    } catch (err) {
+      if (err instanceof SemaphoreLockTimeoutError) {
+        logger.debug(
+          `A certificate for ${name} is being ordered already: not ordering another`,
+          { fullDomain: name } as LogAttributes,
+        );
+      } else {
+        logger.error(
+          `Could not take the certificate order lock of ${name}: not ordering now`,
+          { fullDomain: name } as LogAttributes,
+        );
+        logger.error(err, { fullDomain: name } as LogAttributes);
+      }
+
+      return null;
+    }
+  }
+
+  // Whether this handle holds the lock of this name right now.
+  public static isHeldFor(
+    lock: CertificateOrderLockHandle | undefined,
+    domain: string,
+  ): boolean {
+    return Boolean(
+      lock &&
+        CertificateOrderLock.held.has(lock) &&
+        lock.domain === CertificateOrderLock.normalizeDomain(domain),
+    );
+  }
+
+  /*
+   * Whether this handle did hold the name's lock, and Redis lost it while it
+   * was held: not a mistake of the caller's, just a lock that is gone.
+   */
+  public static wasLost(
+    lock: CertificateOrderLockHandle | undefined,
+    domain: string,
+  ): boolean {
+    return Boolean(
+      lock &&
+        CertificateOrderLock.lost.has(lock) &&
+        lock.domain === CertificateOrderLock.normalizeDomain(domain),
+    );
+  }
+
+  // Never throws: the lock expires on its own, and an order must not fail on it.
+  public static async release(lock: CertificateOrderLockHandle): Promise<void> {
+    const mutex: SemaphoreMutex | undefined =
+      CertificateOrderLock.held.get(lock);
+
+    if (!mutex) {
+      return;
+    }
+
+    CertificateOrderLock.held.delete(lock);
+
+    try {
+      await Semaphore.release(mutex);
+    } catch (err) {
+      logger.error(err, { fullDomain: lock.domain } as LogAttributes);
+    }
+  }
+}

package/Server/Utils/Greenlock/CertificateOrderOutcome.ts

@@ -0,0 +1,29 @@
+/*
+ * How a request to order a certificate ended, when it did not throw.
+ *
+ * Its own module so that GreenlockUtil, which places every order, and
+ * CertificateOrder, which decides whether a first order is needed, can both
+ * use it without importing each other.
+ */
+export enum CertificateOrderOutcome {
+  // A certificate was ordered and stored.
+  Ordered = "Ordered",
+  /*
+   * The name already had a certificate. It was recorded as ordered and
+   * nothing was ordered.
+   */
+  AlreadyIssued = "AlreadyIssued",
+  /*
+   * Nothing was ordered now: another order for the name was running, or the
+   * lock that keeps orders apart could not be taken. The sweeps try again.
+   */
+  NotOrderedNow = "NotOrderedNow",
+  /*
+   * Nothing was ordered now: this window's orders are used up - the share of
+   * the sweep or the click that asked, or the installation's Let's Encrypt
+   * allowance (CertificateOrderBudget). The next window has new ones.
+   */
+  LimitReached = "LimitReached",
+}
+
+export default CertificateOrderOutcome;

package/Server/Utils/Greenlock/CertificateOwners.ts

@@ -0,0 +1,72 @@
+import { Host } from "../../EnvironmentConfig";
+import DashboardDomainService from "../../Services/DashboardDomainService";
+import StatusPageDomainService from "../../Services/StatusPageDomainService";
+
+export type OwnedDomainsLookup = (
+  domains: Array<string>,
+) => Promise<Array<string>>;
+
+export interface CertificateOwner {
+  name: string;
+  // Which of these domains this owner has a certificate in AcmeCertificate for.
+  getOwnedDomains: OwnedDomainsLookup;
+}
+
+/*
+ * Every kind of owner of the certificates in AcmeCertificate.
+ *
+ * Each owner renews and removes its own certificates through
+ * GreenlockUtil.renewAllCertsWhichAreExpiringSoon with its own lookup; this
+ * list is for the one job that has to know about all of them at once, the
+ * cleanup of certificates nobody owns
+ * (GreenlockUtil.removeExpiredCertificatesNobodyOwns).
+ *
+ * A new kind of custom domain that orders certificates must be added here.
+ * Until it is, the cleanup takes its certificates for abandoned ones and
+ * deletes them a month after they expire.
+ */
+export default class CertificateOwners {
+  // The installation's own HOST, without a port, as CoreSSL orders it.
+  public static getPrimaryHostname(host: string = Host): string {
+    return (host.trim().toLowerCase().split(":")[0] || "").trim();
+  }
+
+  public static getAll(): Array<CertificateOwner> {
+    return [
+      {
+        name: "status page domains",
+        getOwnedDomains: async (
+          domains: Array<string>,
+        ): Promise<Array<string>> => {
+          return await StatusPageDomainService.getOwnedDomains(domains);
+        },
+      },
+      {
+        name: "dashboard domains",
+        getOwnedDomains: async (
+          domains: Array<string>,
+        ): Promise<Array<string>> => {
+          return await DashboardDomainService.getOwnedDomains(domains);
+        },
+      },
+      {
+        // Ordered and renewed by the CoreSSL job.
+        name: "primary host",
+        getOwnedDomains: async (
+          domains: Array<string>,
+        ): Promise<Array<string>> => {
+          const primaryHostname: string =
+            CertificateOwners.getPrimaryHostname();
+
+          if (!primaryHostname) {
+            return [];
+          }
+
+          return domains.filter((domain: string) => {
+            return domain === primaryHostname;
+          });
+        },
+      },
+    ];
+  }
+}