@oneuptime/common 11.6.2 → 11.7.0

package/Tests/Utils/Slo/SloHealth.test.ts

@@ -0,0 +1,334 @@
+import SliType from "../../../Types/ServiceLevelObjective/SliType";
+import SloMultiMonitorMode from "../../../Types/ServiceLevelObjective/SloMultiMonitorMode";
+import SloStatus from "../../../Types/ServiceLevelObjective/SloStatus";
+import SloWindowType from "../../../Types/ServiceLevelObjective/SloWindowType";
+import {
+  DEFAULT_AT_RISK_THRESHOLD_PERCENTAGE,
+  getSloBudgetTier,
+  getSloNotice,
+  isRollingWindowNotYetFull,
+  SloBudgetTier,
+  SloNotice,
+  SloNoticeType,
+} from "../../../Utils/Slo/SloHealth";
+
+const SECONDS_PER_DAY: number = 24 * 60 * 60;
+
+/** The full 30-day error budget of a 99.9% SLO: 43m 12s. */
+const THIRTY_DAY_999_BUDGET_SECONDS: number = 0.001 * 30 * SECONDS_PER_DAY;
+
+describe("SloHealth", () => {
+  describe("getSloBudgetTier", () => {
+    it("returns Unknown when the SLO has not been evaluated", () => {
+      expect(getSloBudgetTier({ errorBudgetRemainingPercentage: null })).toBe(
+        SloBudgetTier.Unknown,
+      );
+      expect(
+        getSloBudgetTier({ errorBudgetRemainingPercentage: undefined }),
+      ).toBe(SloBudgetTier.Unknown);
+    });
+
+    it("returns Unknown for non-finite values", () => {
+      expect(
+        getSloBudgetTier({ errorBudgetRemainingPercentage: Number.NaN }),
+      ).toBe(SloBudgetTier.Unknown);
+    });
+
+    it("returns Healthy well above the threshold", () => {
+      expect(getSloBudgetTier({ errorBudgetRemainingPercentage: 68 })).toBe(
+        SloBudgetTier.Healthy,
+      );
+    });
+
+    it("returns Exhausted at exactly zero", () => {
+      expect(getSloBudgetTier({ errorBudgetRemainingPercentage: 0 })).toBe(
+        SloBudgetTier.Exhausted,
+      );
+    });
+
+    it("returns Exhausted when overspent", () => {
+      expect(getSloBudgetTier({ errorBudgetRemainingPercentage: -37 })).toBe(
+        SloBudgetTier.Exhausted,
+      );
+    });
+
+    it("returns AtRisk at exactly the threshold", () => {
+      expect(
+        getSloBudgetTier({
+          errorBudgetRemainingPercentage: DEFAULT_AT_RISK_THRESHOLD_PERCENTAGE,
+        }),
+      ).toBe(SloBudgetTier.AtRisk);
+    });
+
+    it("falls back to the 20% default when no threshold is given", () => {
+      expect(getSloBudgetTier({ errorBudgetRemainingPercentage: 19.9 })).toBe(
+        SloBudgetTier.AtRisk,
+      );
+      expect(getSloBudgetTier({ errorBudgetRemainingPercentage: 20.1 })).toBe(
+        SloBudgetTier.Healthy,
+      );
+    });
+
+    it("honours a custom at-risk threshold instead of the hardcoded 20", () => {
+      /*
+       * The regression this function exists for: at a 50% threshold the
+       * old `remaining <= 20` check rendered 35% green while the worker
+       * had already moved the SLO to At Risk.
+       */
+      expect(
+        getSloBudgetTier({
+          errorBudgetRemainingPercentage: 35,
+          atRiskThresholdPercentage: 50,
+        }),
+      ).toBe(SloBudgetTier.AtRisk);
+
+      expect(
+        getSloBudgetTier({
+          errorBudgetRemainingPercentage: 15,
+          atRiskThresholdPercentage: 5,
+        }),
+      ).toBe(SloBudgetTier.Healthy);
+    });
+
+    it("treats a zero threshold as 'at risk only once exhausted'", () => {
+      expect(
+        getSloBudgetTier({
+          errorBudgetRemainingPercentage: 0.5,
+          atRiskThresholdPercentage: 0,
+        }),
+      ).toBe(SloBudgetTier.Healthy);
+    });
+  });
+
+  describe("getSloNotice", () => {
+    it("returns null for a normally measuring SLO", () => {
+      expect(
+        getSloNotice({
+          isEnabled: true,
+          sloStatus: SloStatus.Healthy,
+          sliType: SliType.MonitorUptime,
+          monitorCount: 2,
+          targetPercentage: 99.9,
+          lastEvaluatedAt: new Date(),
+        }),
+      ).toBeNull();
+    });
+
+    it("reports a disabled SLO ahead of any other reason", () => {
+      /*
+       * A disabled SLO is never evaluated, so its stale Misconfigured
+       * status must not send the user off to attach monitors.
+       */
+      const notice: SloNotice | null = getSloNotice({
+        isEnabled: false,
+        sloStatus: SloStatus.Misconfigured,
+        monitorCount: 0,
+      });
+
+      expect(notice?.type).toBe(SloNoticeType.Info);
+      expect(notice?.title).toBe("This SLO is disabled");
+    });
+
+    it("names 'no monitors' as the reason for Misconfigured", () => {
+      const notice: SloNotice | null = getSloNotice({
+        isEnabled: true,
+        sloStatus: SloStatus.Misconfigured,
+        sliType: SliType.MonitorUptime,
+        monitorCount: 0,
+        targetPercentage: 99.9,
+        lastEvaluatedAt: new Date(),
+      });
+
+      expect(notice?.type).toBe(SloNoticeType.Warning);
+      expect(notice?.title).toBe("No monitors attached");
+    });
+
+    it("names an out-of-range target as the reason for Misconfigured", () => {
+      const notice: SloNotice | null = getSloNotice({
+        isEnabled: true,
+        sloStatus: SloStatus.Misconfigured,
+        sliType: SliType.MonitorUptime,
+        monitorCount: 1,
+        targetPercentage: 100,
+        lastEvaluatedAt: new Date(),
+      });
+
+      expect(notice?.title).toBe("The target is out of range");
+    });
+
+    it("names an unsupported SLI type ahead of the monitor count", () => {
+      const notice: SloNotice | null = getSloNotice({
+        isEnabled: true,
+        sloStatus: SloStatus.Misconfigured,
+        sliType: SliType.Metric,
+        monitorCount: 0,
+        targetPercentage: 99.9,
+        lastEvaluatedAt: new Date(),
+      });
+
+      expect(notice?.body).toContain(SliType.MonitorUptime);
+    });
+
+    it("falls back to a generic reason for Misconfigured", () => {
+      const notice: SloNotice | null = getSloNotice({
+        isEnabled: true,
+        sloStatus: SloStatus.Misconfigured,
+        sliType: SliType.MonitorUptime,
+        monitorCount: 1,
+        targetPercentage: 99.9,
+        lastEvaluatedAt: new Date(),
+      });
+
+      expect(notice?.title).toBe("This SLO cannot be evaluated");
+    });
+
+    it("explains Paused", () => {
+      const notice: SloNotice | null = getSloNotice({
+        isEnabled: true,
+        sloStatus: SloStatus.Paused,
+        sliType: SliType.MonitorUptime,
+        monitorCount: 2,
+        targetPercentage: 99.9,
+        lastEvaluatedAt: new Date(),
+      });
+
+      expect(notice?.type).toBe(SloNoticeType.Info);
+      expect(notice?.title).toBe("Measurement is paused");
+    });
+
+    it("explains a never-evaluated SLO", () => {
+      const notice: SloNotice | null = getSloNotice({
+        isEnabled: true,
+        sloStatus: SloStatus.Healthy,
+        sliType: SliType.MonitorUptime,
+        monitorCount: 1,
+        targetPercentage: 99.9,
+        lastEvaluatedAt: null,
+      });
+
+      expect(notice?.title).toBe("Not evaluated yet");
+    });
+
+    it("says nothing about a healthy SLO whose isEnabled was not selected", () => {
+      // `undefined` means "not loaded", which is not the same as `false`.
+      expect(
+        getSloNotice({
+          sloStatus: SloStatus.Healthy,
+          sliType: SliType.MonitorUptime,
+          monitorCount: 1,
+          targetPercentage: 99.9,
+          lastEvaluatedAt: new Date(),
+        }),
+      ).toBeNull();
+    });
+  });
+
+  describe("isRollingWindowNotYetFull", () => {
+    it("is false for a mature rolling window", () => {
+      expect(
+        isRollingWindowNotYetFull({
+          windowType: SloWindowType.Rolling,
+          windowDays: 30,
+          targetPercentage: 99.9,
+          errorBudgetTotalSeconds: THIRTY_DAY_999_BUDGET_SECONDS,
+        }),
+      ).toBe(false);
+    });
+
+    it("is true for a 30-day SLO carrying only a week of data", () => {
+      expect(
+        isRollingWindowNotYetFull({
+          windowType: SloWindowType.Rolling,
+          windowDays: 30,
+          targetPercentage: 99.9,
+          errorBudgetTotalSeconds: 0.001 * 7 * SECONDS_PER_DAY,
+        }),
+      ).toBe(true);
+    });
+
+    it("stays silent for Monitor Seconds Average — its budget scales with the monitor count", () => {
+      /*
+       * A five-monitor SLO with a week of data carries 35 monitor-days of
+       * budget, which would read as mature against a 30-day yardstick. The
+       * helper declines to guess rather than answer wrongly in either
+       * direction.
+       */
+      expect(
+        isRollingWindowNotYetFull({
+          windowType: SloWindowType.Rolling,
+          windowDays: 30,
+          targetPercentage: 99.9,
+          errorBudgetTotalSeconds: 0.001 * 7 * SECONDS_PER_DAY,
+          multiMonitorMode: SloMultiMonitorMode.MonitorSecondsAverage,
+        }),
+      ).toBe(false);
+    });
+
+    it("still answers for the default Any Monitor Down mode", () => {
+      expect(
+        isRollingWindowNotYetFull({
+          windowType: SloWindowType.Rolling,
+          windowDays: 30,
+          targetPercentage: 99.9,
+          errorBudgetTotalSeconds: 0.001 * 7 * SECONDS_PER_DAY,
+          multiMonitorMode: SloMultiMonitorMode.AnyDown,
+        }),
+      ).toBe(true);
+    });
+
+    it("tolerates the small shortfall between now and the last evaluation", () => {
+      // 0.5% short — a mature SLO must not flicker the banner.
+      expect(
+        isRollingWindowNotYetFull({
+          windowType: SloWindowType.Rolling,
+          windowDays: 30,
+          targetPercentage: 99.9,
+          errorBudgetTotalSeconds: THIRTY_DAY_999_BUDGET_SECONDS * 0.995,
+        }),
+      ).toBe(false);
+    });
+
+    it("is false for calendar-month windows — their budget is never prorated", () => {
+      expect(
+        isRollingWindowNotYetFull({
+          windowType: SloWindowType.CalendarMonth,
+          windowDays: 30,
+          targetPercentage: 99.9,
+          errorBudgetTotalSeconds: 1,
+        }),
+      ).toBe(false);
+    });
+
+    it("is false when the SLO has not been evaluated", () => {
+      expect(
+        isRollingWindowNotYetFull({
+          windowType: SloWindowType.Rolling,
+          windowDays: 30,
+          targetPercentage: 99.9,
+          errorBudgetTotalSeconds: null,
+        }),
+      ).toBe(false);
+    });
+
+    it("is false when the target would make the full-window budget meaningless", () => {
+      expect(
+        isRollingWindowNotYetFull({
+          windowType: SloWindowType.Rolling,
+          windowDays: 30,
+          targetPercentage: 100,
+          errorBudgetTotalSeconds: 60,
+        }),
+      ).toBe(false);
+    });
+
+    it("defaults windowDays to 30 when it is not set", () => {
+      expect(
+        isRollingWindowNotYetFull({
+          windowType: SloWindowType.Rolling,
+          targetPercentage: 99.9,
+          errorBudgetTotalSeconds: 0.001 * 7 * SECONDS_PER_DAY,
+        }),
+      ).toBe(true);
+    });
+  });
+});

package/Utils/Slo/SloBurnRateRuleState.ts

@@ -0,0 +1,91 @@
+/**
+ * Whether a burn rate rule currently has an alert open.
+ *
+ * `lastAlertCreatedAt` / `lastAlertResolvedAt` are stamped by the
+ * evaluation worker every time it fires or resolves a rule's alert, and
+ * together they are the only record of the rule's lifecycle — but nothing
+ * read them outside the worker, so the Burn Rate Rules table could not
+ * tell the user whether a rule was paging someone right now, had ever
+ * fired, or had never once triggered.
+ *
+ * The predicate is the worker's own (EvaluateSlos.evaluateBurnRateRule's
+ * `hasUnresolvedFiringState`), lifted here so the table and the worker can
+ * never disagree about what "firing" means.
+ *
+ * Kept free of React and of the database model so both can import it and
+ * so the ordering edges are unit-testable without a DOM.
+ */
+
+import SloStatus from "../../Types/ServiceLevelObjective/SloStatus";
+
+export interface SloBurnRateRuleAlertState {
+  lastAlertCreatedAt?: Date | undefined | null;
+  lastAlertResolvedAt?: Date | undefined | null;
+}
+
+/**
+ * True when the rule has fired and that alert has not been resolved
+ * since.
+ *
+ * A resolve stamped at exactly the same instant as the fire counts as
+ * resolved: the worker writes the resolve strictly after the fire, so an
+ * equal pair can only be the same lifecycle, and treating it as firing
+ * would leave a red "Firing" pill on a rule with nothing open.
+ */
+export type IsBurnRateRuleFiringFunction = (
+  rule: SloBurnRateRuleAlertState,
+) => boolean;
+
+export const isBurnRateRuleFiring: IsBurnRateRuleFiringFunction = (
+  rule: SloBurnRateRuleAlertState,
+): boolean => {
+  if (!rule.lastAlertCreatedAt) {
+    return false;
+  }
+
+  if (!rule.lastAlertResolvedAt) {
+    return true;
+  }
+
+  return rule.lastAlertResolvedAt.getTime() < rule.lastAlertCreatedAt.getTime();
+};
+
+/**
+ * Whether the OWNING SLO is in a state where a burn rate rule could be
+ * firing at all.
+ *
+ * `isBurnRateRuleFiring` alone is not enough to render a live "Firing"
+ * badge. When an SLO is disabled, or the worker's guards move it to
+ * Misconfigured or Paused, the SLO-level resolve paths
+ * (ServiceLevelObjectiveService.resolveOpenBurnRateAlertsForSlo and the
+ * worker's setGuardStatusAndResolveOpenAlerts) close every open alert but
+ * deliberately do NOT stamp `lastAlertResolvedAt` — that column drives a
+ * live rule's re-fire suppression, and this is a state change of the SLO
+ * rather than a burn rate that recovered. The rule's own columns are
+ * therefore left mid-lifecycle, and a UI reading them alone would show a
+ * red "Firing" pill on a rule with nothing open, forever.
+ *
+ * Pairing the two predicates keeps the badge honest without touching the
+ * suppression semantics the worker depends on.
+ */
+export interface SloBurnRateFiringContext {
+  isEnabled?: boolean | undefined | null;
+  sloStatus?: SloStatus | undefined | null;
+}
+
+export type CanSloFireBurnRateAlertsFunction = (
+  slo: SloBurnRateFiringContext,
+) => boolean;
+
+export const canSloFireBurnRateAlerts: CanSloFireBurnRateAlertsFunction = (
+  slo: SloBurnRateFiringContext,
+): boolean => {
+  if (slo.isEnabled === false) {
+    return false;
+  }
+
+  return (
+    slo.sloStatus !== SloStatus.Misconfigured &&
+    slo.sloStatus !== SloStatus.Paused
+  );
+};

package/Utils/Slo/SloDuration.ts

@@ -0,0 +1,165 @@
+/**
+ * Compact duration rendering for error budgets.
+ *
+ * Error budgets span four orders of magnitude — a 99.99% / 7-day SLO gets
+ * 60 seconds, a 99% / 90-day SLO gets 21.6 hours — so the "minutes only"
+ * rendering the SLO pages started with reads badly at both ends: a
+ * one-minute budget shows as "1 min", and a 90-day budget shows as
+ * "1296 min of 1296 min". The docs quote budgets the way an SRE says them
+ * out loud ("43m 12s", "2h 9m"), and this is the helper that produces
+ * exactly that.
+ *
+ * Kept free of React (and of the database models) so the SLO pages, the
+ * dashboard SLO widget and any future status-page surface can all share
+ * one rendering, and so every rounding edge is unit-testable without a
+ * DOM.
+ */
+
+/** U+2212 MINUS SIGN — reads as a minus rather than a hyphen at tile sizes. */
+const MINUS_SIGN: string = "−";
+
+/**
+ * Seconds per unit, largest first. Only the two most significant non-zero
+ * units are rendered: "2d 5h" is what an on-call engineer needs, "2d 5h
+ * 13m 7s" is noise, and truncating rather than rounding up keeps the
+ * remaining budget from ever reading larger than it is.
+ */
+const UNITS: Array<{ suffix: string; seconds: number }> = [
+  { suffix: "d", seconds: 86400 },
+  { suffix: "h", seconds: 3600 },
+  { suffix: "m", seconds: 60 },
+  { suffix: "s", seconds: 1 },
+];
+
+const MAX_UNITS_SHOWN: number = 2;
+
+/**
+ * `null` for anything that is not a real, finite number. The
+ * worker-owned budget columns are NULL until the SLO is first evaluated,
+ * and a Postgres decimal can arrive as a string over JSON, so the guard
+ * has to reject non-numbers rather than coerce them.
+ */
+type ToFiniteNumberFunction = (
+  value: number | undefined | null,
+) => number | null;
+
+const toFiniteNumber: ToFiniteNumberFunction = (
+  value: number | undefined | null,
+): number | null => {
+  if (typeof value !== "number" || !isFinite(value)) {
+    return null;
+  }
+
+  return value;
+};
+
+/**
+ * An unsigned duration as "2d 5h" / "43m 12s" / "45s". Fractional seconds
+ * are floored, so a 59.9-second budget renders "59s" rather than rounding
+ * up to a minute the SLO does not actually have.
+ *
+ * A duration that floors to zero renders "0s" — a real, sayable state —
+ * instead of an empty string.
+ */
+export type FormatDurationCompactFunction = (seconds: number) => string;
+
+export const formatDurationCompact: FormatDurationCompactFunction = (
+  seconds: number,
+): string => {
+  let remainingSeconds: number = Math.floor(Math.abs(seconds));
+
+  const parts: Array<string> = [];
+
+  for (const unit of UNITS) {
+    if (parts.length >= MAX_UNITS_SHOWN) {
+      break;
+    }
+
+    const count: number = Math.floor(remainingSeconds / unit.seconds);
+
+    /*
+     * Skip leading zero units ("0d 5h"), but once a larger unit has been
+     * emitted a zero is still skipped rather than padded — "2d" is
+     * correct for exactly 48 hours, "2d 0h" is not how anyone says it.
+     */
+    if (count <= 0) {
+      continue;
+    }
+
+    parts.push(`${count}${unit.suffix}`);
+    remainingSeconds -= count * unit.seconds;
+  }
+
+  if (parts.length === 0) {
+    return "0s";
+  }
+
+  return parts.join(" ");
+};
+
+/**
+ * `errorBudgetRemainingSeconds` is SIGNED: negative means the budget is
+ * overspent, and that overage is the single most useful number during a
+ * bad month. Render it plainly rather than clamping to zero, which would
+ * hide a breach.
+ *
+ * Returns `null` (not a placeholder string) when the SLO has not been
+ * evaluated, so callers decide how to phrase "no data yet" in their own
+ * context.
+ */
+export type FormatErrorBudgetRemainingFunction = (
+  seconds: number | undefined | null,
+) => string | null;
+
+export const formatErrorBudgetRemaining: FormatErrorBudgetRemainingFunction = (
+  seconds: number | undefined | null,
+): string | null => {
+  const numericSeconds: number | null = toFiniteNumber(seconds);
+
+  if (numericSeconds === null) {
+    return null;
+  }
+
+  if (numericSeconds < 0) {
+    return `${MINUS_SIGN}${formatDurationCompact(numericSeconds)} over budget`;
+  }
+
+  return `${formatDurationCompact(numericSeconds)} left`;
+};
+
+/**
+ * "12m 30s left of 43m 12s" — the phrasing the error-budget docs use.
+ * Falls back to the bare remaining string when the total is not known
+ * (the two columns are written together by the worker, but a partially
+ * evaluated row must not render "of null").
+ */
+export type FormatErrorBudgetRemainingOfTotalFunction = (data: {
+  remainingSeconds: number | undefined | null;
+  totalSeconds: number | undefined | null;
+}) => string | null;
+
+export const formatErrorBudgetRemainingOfTotal: FormatErrorBudgetRemainingOfTotalFunction =
+  (data: {
+    remainingSeconds: number | undefined | null;
+    totalSeconds: number | undefined | null;
+  }): string | null => {
+    const remainingText: string | null = formatErrorBudgetRemaining(
+      data.remainingSeconds,
+    );
+
+    if (remainingText === null) {
+      return null;
+    }
+
+    const totalSeconds: number | null = toFiniteNumber(data.totalSeconds);
+
+    /*
+     * A zero total is a real state for a brand-new SLO (no elapsed window
+     * yet). "of 0s" adds nothing, so drop the suffix rather than print it.
+     */
+    if (totalSeconds === null || totalSeconds <= 0) {
+      return remainingText;
+    }
+
+    return `${remainingText} of ${formatDurationCompact(totalSeconds)}`;
+  };

package/Utils/Slo/SloEvaluation.ts

@@ -0,0 +1,22 @@
+/**
+ * Cadence constants shared by the SLO evaluation worker and every UI
+ * surface that describes what the worker produced.
+ *
+ * These numbers were previously duplicated as prose: the worker owned
+ * them, and the dashboard said "evaluated every few minutes" and "1×
+ * spends the budget exactly over the window" without ever naming the
+ * lookback the headline burn rate is actually measured over. A reader who
+ * compared the tile against a burn rate rule had no way to know they were
+ * measuring different spans. Naming them once here keeps the copy and the
+ * computation from drifting.
+ */
+
+/** How often a due SLO is re-evaluated, and how far ahead the worker schedules the next run. */
+export const SLO_EVALUATION_CADENCE_MINUTES: number = 5;
+
+/**
+ * Lookback for the "current burn rate" column shown on the SLO overview
+ * and the dashboard SLO widget. Deliberately equal to the default
+ * fast-burn rule's long window, so the tile and that rule move together.
+ */
+export const SLO_CURRENT_BURN_RATE_WINDOW_MINUTES: number = 60;