@oneuptime/common 12.0.28 → 12.0.30

package/Server/Utils/Greenlock/Greenlock.ts

@@ -16,11 +16,64 @@ import ServerException from "../../../Types/Exception/ServerException";
 import Text from "../../../Types/Text";
 import AcmeCertificate from "../../../Models/DatabaseModels/AcmeCertificate";
 import AcmeChallenge from "../../../Models/DatabaseModels/AcmeChallenge";
+import ArrayUtil from "../../../Utils/Array";
 import acme from "acme-client";
 import { Challenge } from "acme-client/types/rfc8555";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 
 export default class GreenlockUtil {
+  /*
+   * How early a certificate is renewed, as a range rather than a single value.
+   *
+   * Let's Encrypt issues for 90 days, so every domain ordered on the same day
+   * expires on the same day. Renewing them all at one fixed lead time keeps
+   * that batch together for every cycle that follows, and a single bad day -
+   * an upstream outage, a rate limit, a run that does not finish - then expires
+   * the whole batch at once instead of one domain.
+   *
+   * Spreading the lead time across a range pulls the batch apart: domains that
+   * expire on the same day come due days apart, and having been renewed days
+   * apart they expire days apart next cycle too. The offset is derived from the
+   * domain name so it is stable - a domain must not drift in and out of
+   * eligibility between two runs, which would leave it renewed by neither.
+   */
+  public static readonly RENEW_LEAD_TIME_MAX_IN_DAYS: number = 40;
+  public static readonly RENEW_LEAD_TIME_MIN_IN_DAYS: number = 25;
+
+  /*
+   * A run renews at most this many domains, at most this many at a time.
+   *
+   * Both bound how much of a backlog is dispatched at once. Let's Encrypt
+   * allows 300 new orders per account per three hours, and the reactive
+   * provisioning sweep spends from the same allowance, so a run that tried to
+   * clear a large backlog in one pass would spend the account's budget and get
+   * the renewals behind it refused. At the schedule this job runs on the cap
+   * still clears far more per day than one sequential daily pass ever did,
+   * while leaving room under the limit.
+   */
+  public static readonly RENEW_MAX_PER_RUN: number = 10;
+  public static readonly RENEW_CONCURRENCY: number = 5;
+
+  /*
+   * Stable per-domain lead time, somewhere in the range above. Same domain,
+   * same answer, on every run and every replica.
+   */
+  public static getRenewalLeadTimeInDays(domain: string): number {
+    const spanInDays: number =
+      GreenlockUtil.RENEW_LEAD_TIME_MAX_IN_DAYS -
+      GreenlockUtil.RENEW_LEAD_TIME_MIN_IN_DAYS;
+
+    let hash: number = 0;
+
+    for (let i: number = 0; i < domain.length; i++) {
+      hash = (hash * 31 + domain.charCodeAt(i)) % 1000003;
+    }
+
+    return (
+      GreenlockUtil.RENEW_LEAD_TIME_MIN_IN_DAYS + (hash % (spanInDays + 1))
+    );
+  }
+
   @CaptureSpan()
   public static async renewAllCertsWhichAreExpiringSoon(data: {
     validateCname: (domain: string) => Promise<boolean>;
@@ -29,15 +82,17 @@ export default class GreenlockUtil {
     try {
       logger.debug("Renewing all certificates");
 
-      // get all certificates which are expiring soon
-
+      /*
+       * Read the widest window any domain's lead time can make it due in, then
+       * keep only the domains whose own lead time has actually been reached.
+       */
       const certificates: AcmeCertificate[] =
         await AcmeCertificateService.findBy({
           query: {
             expiresAt: QueryHelper.lessThanEqualTo(
               OneUptimeDate.addRemoveDays(
                 OneUptimeDate.getCurrentDate(),
-                40, // 40 days before expiry
+                GreenlockUtil.RENEW_LEAD_TIME_MAX_IN_DAYS,
               ),
             ),
           },
@@ -45,6 +100,7 @@ export default class GreenlockUtil {
           skip: 0,
           select: {
             domain: true,
+            expiresAt: true,
           },
           sort: {
             expiresAt: SortOrder.Ascending,
@@ -54,76 +110,118 @@ export default class GreenlockUtil {
           },
         });
 
+      const now: Date = OneUptimeDate.getCurrentDate();
+
+      const dueCertificates: AcmeCertificate[] = certificates.filter(
+        (certificate: AcmeCertificate) => {
+          if (!certificate.domain || !certificate.expiresAt) {
+            return false;
+          }
+
+          const renewAt: Date = OneUptimeDate.addRemoveDays(
+            certificate.expiresAt,
+            -GreenlockUtil.getRenewalLeadTimeInDays(certificate.domain),
+          );
+
+          return !OneUptimeDate.isAfter(renewAt, now);
+        },
+      );
+
+      /*
+       * Still sorted by expiry, so a run that cannot take the whole backlog
+       * spends itself on the domains closest to expiring and leaves the rest -
+       * which by construction still have weeks of lead time - to the next run.
+       */
+      const batch: AcmeCertificate[] = dueCertificates.slice(
+        0,
+        GreenlockUtil.RENEW_MAX_PER_RUN,
+      );
+
       logger.debug(
-        `Found ${certificates.length} certificates which are expiring soon`,
-        { certificateCount: certificates.length },
+        `Found ${dueCertificates.length} certificates due for renewal, renewing ${batch.length} in this run`,
+        {
+          dueCount: dueCertificates.length,
+          batchCount: batch.length,
+        },
+      );
+
+      await ArrayUtil.forEachWithConcurrency(
+        batch,
+        GreenlockUtil.RENEW_CONCURRENCY,
+        async (certificate: AcmeCertificate): Promise<void> => {
+          await GreenlockUtil.renewCertForDomain({
+            domain: certificate.domain as string,
+            validateCname: data.validateCname,
+            notifyDomainRemoved: data.notifyDomainRemoved,
+          });
+        },
       );
+    } catch (e) {
+      logger.error("Error renewing all certificates");
+      logger.error(e);
 
-      // order certificate for each domain
+      throw e;
+    }
+  }
 
-      for (const certificate of certificates) {
-        if (!certificate.domain) {
-          continue;
-        }
+  /*
+   * Renew one domain. Never throws: one domain that cannot be renewed - a CNAME
+   * that no longer points here, a challenge the CA would not accept - must not
+   * take down the rest of the run with it.
+   */
+  private static async renewCertForDomain(data: {
+    domain: string;
+    validateCname: (domain: string) => Promise<boolean>;
+    notifyDomainRemoved: (domain: string) => Promise<void>;
+  }): Promise<void> {
+    const { domain } = data;
+
+    const certLogAttributes: LogAttributes = {
+      domain: domain,
+    };
 
-        const certLogAttributes: LogAttributes = {
-          domain: certificate.domain,
-        };
+    logger.debug(
+      `Renewing certificate for domain: ${domain}`,
+      certLogAttributes,
+    );
 
+    try {
+      //validate cname
+      const isValidCname: boolean = await data.validateCname(domain);
+
+      if (!isValidCname) {
         logger.debug(
-          `Renewing certificate for domain: ${certificate.domain}`,
+          `CNAME is not valid for domain: ${domain}`,
           certLogAttributes,
         );
 
-        try {
-          //validate cname
-          const isValidCname: boolean = await data.validateCname(
-            certificate.domain,
-          );
-
-          if (!isValidCname) {
-            logger.debug(
-              `CNAME is not valid for domain: ${certificate.domain}`,
-              certLogAttributes,
-            );
-
-            // if cname is not valid then remove the domain
-            await GreenlockUtil.removeDomain(certificate.domain);
-            await data.notifyDomainRemoved(certificate.domain);
+        // if cname is not valid then remove the domain
+        await GreenlockUtil.removeDomain(domain);
+        await data.notifyDomainRemoved(domain);
 
-            logger.error(
-              `Cname is not valid for domain: ${certificate.domain}`,
-              certLogAttributes,
-            );
-          } else {
-            logger.debug(
-              `CNAME is valid for domain: ${certificate.domain}`,
-              certLogAttributes,
-            );
+        logger.error(
+          `Cname is not valid for domain: ${domain}`,
+          certLogAttributes,
+        );
+      } else {
+        logger.debug(`CNAME is valid for domain: ${domain}`, certLogAttributes);
 
-            await GreenlockUtil.orderCert({
-              domain: certificate.domain,
-              validateCname: data.validateCname,
-            });
+        await GreenlockUtil.orderCert({
+          domain: domain,
+          validateCname: data.validateCname,
+        });
 
-            logger.debug(
-              `Certificate renewed for domain: ${certificate.domain}`,
-              certLogAttributes,
-            );
-          }
-        } catch (e) {
-          logger.error(
-            `Error renewing certificate for domain: ${certificate.domain}`,
-            certLogAttributes,
-          );
-          logger.error(e, certLogAttributes);
-        }
+        logger.debug(
+          `Certificate renewed for domain: ${domain}`,
+          certLogAttributes,
+        );
       }
     } catch (e) {
-      logger.error("Error renewing all certificates");
-      logger.error(e);
-
-      throw e;
+      logger.error(
+        `Error renewing certificate for domain: ${domain}`,
+        certLogAttributes,
+      );
+      logger.error(e, certLogAttributes);
     }
   }
 

package/Server/Utils/LogPipelineProcessorValidation.ts

@@ -0,0 +1,112 @@
+import BadDataException from "../../Types/Exception/BadDataException";
+import { JSONObject } from "../../Types/JSON";
+import LogPipelineProcessorType, {
+  GrokParserConfig,
+} from "../../Types/Log/LogPipelineProcessorType";
+import { compileGrokPattern } from "../../Utils/Grok/Grok";
+
+/*
+ * Save-time validation for log pipeline processors.
+ *
+ * A processor that cannot do anything is worse than a rejected one: it
+ * sits in the pipeline list looking configured while every log flows
+ * past it untouched, and nothing surfaces the fact. That is exactly how
+ * GrokParser behaved before it was implemented
+ * (OneUptime/oneuptime#2515).
+ *
+ * The grok pattern is the one field where a typo is both easy and
+ * invisible - `%{IPV4:client ip}` or a stray bracket compiles to
+ * nothing. Compiling it here is cheap, happens once per human edit, and
+ * puts the error in front of the only person who can fix it. The ingest
+ * path stays defensive anyway: rows saved before this validation
+ * existed still fail closed to "leave the log alone".
+ */
+
+/*
+ * Prefixes become the leading part of an attribute key, so hold them to
+ * what an attribute key can sensibly be.
+ */
+const TARGET_PREFIX_REGEX: RegExp = /^[A-Za-z_][A-Za-z0-9_.@:-]*$/;
+
+export interface LogPipelineProcessorCandidate {
+  processorType: string | undefined | null;
+  configuration: JSONObject | string | undefined | null;
+}
+
+/*
+ * `configuration` is a jsonb column, but the dashboard's JSON form field
+ * has historically persisted it as a JSON string literal - so both
+ * shapes reach here (see LogPipelineService.normalizeProcessorConfig).
+ */
+function readConfiguration(
+  configuration: JSONObject | string | undefined | null,
+): JSONObject | null {
+  if (configuration && typeof configuration === "object") {
+    return configuration as JSONObject;
+  }
+
+  if (typeof configuration === "string") {
+    try {
+      const parsed: unknown = JSON.parse(configuration);
+
+      if (parsed && typeof parsed === "object") {
+        return parsed as JSONObject;
+      }
+    } catch {
+      throw new BadDataException("Processor configuration is not valid JSON.");
+    }
+  }
+
+  return null;
+}
+
+export function validateLogPipelineProcessor(
+  candidate: LogPipelineProcessorCandidate,
+): void {
+  if (candidate.processorType !== LogPipelineProcessorType.GrokParser) {
+    return;
+  }
+
+  const configuration: JSONObject | null = readConfiguration(
+    candidate.configuration,
+  );
+
+  if (!configuration) {
+    throw new BadDataException(
+      "A Grok Parser processor needs a configuration with a grok pattern.",
+    );
+  }
+
+  const config: GrokParserConfig = configuration as unknown as GrokParserConfig;
+  const pattern: string =
+    typeof config.pattern === "string" ? config.pattern.trim() : "";
+
+  if (!pattern) {
+    throw new BadDataException(
+      "A Grok Parser processor needs a grok pattern, for example: %{IPV4:client_ip} %{WORD:verb}",
+    );
+  }
+
+  // Throws BadDataException with a message written for the person editing.
+  compileGrokPattern(pattern);
+
+  if (config.targetPrefix !== undefined && config.targetPrefix !== null) {
+    if (typeof config.targetPrefix !== "string") {
+      throw new BadDataException("Target prefix must be text.");
+    }
+
+    const targetPrefix: string = config.targetPrefix.trim();
+
+    if (targetPrefix && !TARGET_PREFIX_REGEX.test(targetPrefix)) {
+      throw new BadDataException(
+        `"${config.targetPrefix}" is not a valid target prefix. Use letters, digits, and . _ - : @ (starting with a letter or underscore).`,
+      );
+    }
+  }
+
+  if (config.source !== undefined && config.source !== null) {
+    if (typeof config.source !== "string") {
+      throw new BadDataException("Source field must be text.");
+    }
+  }
+}

package/Server/Utils/OnCall/CalendarFeedToken.ts

@@ -0,0 +1,241 @@
+import crypto from "crypto";
+import OneUptimeDate from "../../../Types/Date";
+import { PREVIOUS_TOKEN_GRACE_DAYS } from "../../../Types/OnCallDutyPolicy/CalendarFeedWindow";
+
+/*
+ * The capability token that addresses an on-call calendar feed.
+ *
+ * A calendar client keeps the feed URL forever and re-fetches it unattended,
+ * so the token has to be both unguessable and cheap to look up: 256 bits from
+ * the CSPRNG, carried in the URL as base64url (43 characters, no padding), and
+ * stored ONLY as its unkeyed SHA-256 hex digest. The digest is what the public
+ * route queries by, which keeps the plaintext out of the indexed column; a
+ * database dump therefore yields hashes, not working feed URLs.
+ *
+ * Unkeyed on purpose: a keyed hash (HMAC over EncryptionSecret, the backup-code
+ * recipe) would tie every feed to the instance secret, and rotating that secret
+ * would silently break every subscribed calendar in the world with no way to
+ * tell the user. The token already carries 256 bits of entropy, so a rainbow
+ * table buys nothing and the key adds no strength worth that failure mode.
+ *
+ * The hint is the last four characters - enough for the settings page to say
+ * "link ending in …k3Qx" so a user can tell two links apart, far too little to
+ * recover anything.
+ */
+
+// crypto.randomBytes(32) -> 32 bytes -> ceil(32 * 8 / 6) = 43 base64url chars.
+const TOKEN_BYTE_LENGTH: number = 32;
+
+export const CALENDAR_FEED_TOKEN_LENGTH: number = 43;
+
+export const CALENDAR_FEED_TOKEN_HINT_LENGTH: number = 4;
+
+/*
+ * Shape guard for the public route. Anything that does not match is answered
+ * 404 before any database work, so a scanner probing the path costs one regex.
+ */
+export const CALENDAR_FEED_TOKEN_REGEX: RegExp = /^[A-Za-z0-9_-]{43}$/;
+
+export interface MintedCalendarFeedToken {
+  /** The plaintext token. Goes into the URL and, encrypted, into `token`. */
+  token: string;
+  /** Unkeyed SHA-256 hex digest of the token. The indexed lookup column. */
+  tokenHash: string;
+  /** Last four characters of the token, for display only. */
+  tokenHint: string;
+}
+
+/*
+ * Everything a feed row's token columns hold after a rotation: the new
+ * token set, when it was minted, and the rotated-out hash with the instant
+ * it stops being honoured. `previousTokenHash` is null when there was no
+ * token before (a first mint), so a fresh feed never carries a grace entry.
+ */
+export interface CalendarFeedRotation extends MintedCalendarFeedToken {
+  rotatedAt: Date;
+  previousTokenHash: string | null;
+  previousTokenExpiresAt: Date | null;
+}
+
+/*
+ * The token columns every feed model carries (UserOnCallCalendarFeed,
+ * OnCallDutyPolicyScheduleCalendarFeed, ProjectOnCallCalendarFeed). Typed
+ * structurally so the three services share one rule without a common base.
+ */
+export interface CalendarFeedTokenColumns {
+  token?: string | undefined;
+  tokenHash?: string | undefined;
+  tokenHint?: string | undefined;
+  rotatedAt?: Date | undefined;
+  previousTokenHash?: string | undefined;
+  previousTokenExpiresAt?: Date | undefined;
+}
+
+export interface CalendarFeedRotationUpdateData {
+  token: string;
+  tokenHash: string;
+  tokenHint: string;
+  rotatedAt: Date;
+  previousTokenHash: string;
+  previousTokenExpiresAt: Date;
+}
+
+export default class CalendarFeedToken {
+  /**
+   * A fresh 43-character base64url token from 32 CSPRNG bytes.
+   */
+  public static mint(): string {
+    return crypto.randomBytes(TOKEN_BYTE_LENGTH).toString("base64url");
+  }
+
+  /**
+   * Unkeyed SHA-256 of the token, lowercase hex (64 characters). This is the
+   * value stored in `tokenHash` / `previousTokenHash` and the value the public
+   * route looks up by. Deterministic: the same token always hashes the same.
+   */
+  public static hash(token: string): string {
+    return crypto.createHash("sha256").update(token, "utf8").digest("hex");
+  }
+
+  /**
+   * The display hint - the last four characters of the token. Never enough to
+   * reconstruct the token; enough to tell "…k3Qx" from "…Q9zA" in the UI.
+   */
+  public static hint(token: string): string {
+    return token.slice(-CALENDAR_FEED_TOKEN_HINT_LENGTH);
+  }
+
+  /**
+   * True when the string has exactly the shape a minted token has. Used as
+   * the shape guard on the public route so a malformed path never reaches the
+   * database.
+   */
+  public static isValidShape(token: unknown): token is string {
+    return typeof token === "string" && CALENDAR_FEED_TOKEN_REGEX.test(token);
+  }
+
+  /**
+   * Everything a feed row needs when a token is minted or rotated.
+   */
+  public static mintSet(): MintedCalendarFeedToken {
+    const token: string = CalendarFeedToken.mint();
+
+    return {
+      token: token,
+      tokenHash: CalendarFeedToken.hash(token),
+      tokenHint: CalendarFeedToken.hint(token),
+    };
+  }
+
+  /**
+   * The token columns a feed row should hold after minting or rotating.
+   *
+   * The hash the row held before (if any) moves to `previousTokenHash` and
+   * keeps serving an EMPTY calendar until `previousTokenExpiresAt`
+   * (PREVIOUS_TOKEN_GRACE_DAYS from now), so a still-subscribed client clears
+   * its copy instead of showing "could not fetch" until the user notices.
+   */
+  public static buildRotation(data: {
+    currentTokenHash: string | null | undefined;
+    now?: Date | undefined;
+  }): CalendarFeedRotation {
+    const now: Date = data.now || OneUptimeDate.getCurrentDate();
+    const minted: MintedCalendarFeedToken = CalendarFeedToken.mintSet();
+
+    const previousTokenHash: string | null = data.currentTokenHash || null;
+
+    return {
+      ...minted,
+      rotatedAt: now,
+      previousTokenHash: previousTokenHash,
+      previousTokenExpiresAt: previousTokenHash
+        ? OneUptimeDate.addRemoveDays(now, PREVIOUS_TOKEN_GRACE_DAYS)
+        : null,
+    };
+  }
+
+  /**
+   * Populate the token columns of a row about to be INSERTED.
+   *
+   * With `trustSuppliedToken`, a well-formed plaintext token already on the
+   * row (the calendar API mints one up front so it can build the URL in the
+   * same response) is kept and the hash and hint are DERIVED from it. Without
+   * it - every non-root create - whatever the request carried is discarded
+   * and a fresh token is minted. Either way a malformed token or a
+   * caller-chosen hash is replaced: there is no path to a row whose hash
+   * disagrees with its token, and no path for a request body to choose the
+   * secret. A brand-new row never carries a grace-period hash.
+   */
+  public static applyTokenColumnsOnCreate(
+    data: CalendarFeedTokenColumns,
+    options: { trustSuppliedToken: boolean },
+  ): MintedCalendarFeedToken {
+    let minted: MintedCalendarFeedToken;
+
+    if (
+      options.trustSuppliedToken &&
+      CalendarFeedToken.isValidShape(data.token)
+    ) {
+      minted = {
+        token: data.token,
+        tokenHash: CalendarFeedToken.hash(data.token),
+        tokenHint: CalendarFeedToken.hint(data.token),
+      };
+    } else {
+      minted = CalendarFeedToken.mintSet();
+    }
+
+    data.token = minted.token;
+    data.tokenHash = minted.tokenHash;
+    data.tokenHint = minted.tokenHint;
+    data.rotatedAt = OneUptimeDate.getCurrentDate();
+    data.previousTokenHash = undefined;
+    data.previousTokenExpiresAt = undefined;
+
+    return minted;
+  }
+
+  /**
+   * The partial row a rotation writes. `previousTokenHash` /
+   * `previousTokenExpiresAt` are null on a first mint, which the update path
+   * has to be able to write; the model types them as optional strings/dates,
+   * hence the cast at this one boundary.
+   */
+  public static toRotationUpdateData(
+    rotation: CalendarFeedRotation,
+  ): CalendarFeedRotationUpdateData {
+    return {
+      token: rotation.token,
+      tokenHash: rotation.tokenHash,
+      tokenHint: rotation.tokenHint,
+      rotatedAt: rotation.rotatedAt,
+      previousTokenHash: rotation.previousTokenHash as unknown as string,
+      previousTokenExpiresAt:
+        rotation.previousTokenExpiresAt as unknown as Date,
+    };
+  }
+
+  /**
+   * Constant-time comparison of two hashes, for callers that compare a
+   * freshly computed digest against a stored one in memory rather than in a
+   * WHERE clause. Length mismatch is answered false without leaking timing on
+   * the content.
+   */
+  public static isHashEqual(a: string, b: string): boolean {
+    /*
+     * Plain Uint8Arrays rather than Buffers: the two are interchangeable at
+     * runtime, but the App build's @types/node rejects Buffer where an
+     * ArrayBufferView is expected (the same mismatch PasswordHash and
+     * TwoFactorBackupCode trip over), and TextEncoder is global on every
+     * supported Node.
+     */
+    const bytesA: Uint8Array = new TextEncoder().encode(a);
+    const bytesB: Uint8Array = new TextEncoder().encode(b);
+
+    if (bytesA.length !== bytesB.length) {
+      return false;
+    }
+
+    return crypto.timingSafeEqual(bytesA, bytesB);
+  }
+}