@oneuptime/common 13.0.8 → 14.0.1

package/Server/Utils/FrontendEnvironment.ts

@@ -1,10 +1,49 @@
-import { getFrontendEnvVars } from "../EnvironmentConfig";
+import {
+  getFrontendEnvVars,
+  isEnterpriseEditionRequested,
+} from "../EnvironmentConfig";
+import EnterpriseEdition from "../Enterprise/EnterpriseEdition";
 import { ExpressRequest, ExpressResponse } from "./Express";
 import Response from "./Response";
+import { JSONObject } from "../../Types/JSON";
 
 export const FRONTEND_ENVIRONMENT_CACHE_CONTROL: string =
   "private, no-store, no-cache, must-revalidate";
 
+/*
+ * The edition the frontends are told about is the EFFECTIVE one: whether the
+ * enterprise module actually loaded in this process, not what the raw
+ * IS_ENTERPRISE_EDITION variable claims. A Community image with a leftover
+ * IS_ENTERPRISE_EDITION=true must not show enterprise pages whose server
+ * routes do not exist; ENTERPRISE_EDITION_REQUESTED_BUT_NOT_LOADED lets the
+ * admin UI explain that mismatch to a master admin instead.
+ *
+ * "Requested" is EnvironmentConfig's isEnterpriseEditionRequested, the same
+ * definition the App's boot guard uses: an explicit ONEUPTIME_EDITION=community
+ * withdraws the request, so the Enterprise image (which bakes
+ * IS_ENTERPRISE_EDITION=true) run as the Community Edition on purpose is not
+ * told to switch images. It is read on every call, like the rest of the
+ * environment served here.
+ */
+export const getFrontendEnvironmentVariables: () => JSONObject =
+  (): JSONObject => {
+    const frontendEnv: JSONObject = getFrontendEnvVars();
+    const isEnterpriseEditionLoaded: boolean = EnterpriseEdition.isLoaded();
+    const isEnterpriseEditionExpected: boolean = isEnterpriseEditionRequested(
+      process.env,
+    );
+
+    frontendEnv["IS_ENTERPRISE_EDITION"] = isEnterpriseEditionLoaded
+      ? "true"
+      : "false";
+    frontendEnv["ENTERPRISE_EDITION_REQUESTED_BUT_NOT_LOADED"] =
+      isEnterpriseEditionExpected && !isEnterpriseEditionLoaded
+        ? "true"
+        : "false";
+
+    return frontendEnv;
+  };
+
 /*
  * The combined App server and the five standalone frontend servers must emit
  * exactly the same environment payload. Keeping the serializer here prevents
@@ -19,7 +58,7 @@ if(!window.process){
 if(!window.process.env){
   window.process.env = {}
 }
-window.process.env = ${JSON.stringify(getFrontendEnvVars())};
+window.process.env = ${JSON.stringify(getFrontendEnvironmentVariables())};
 `;
 };
 

package/Server/Utils/InstanceHealth/RedisHealth.ts

@@ -2,38 +2,14 @@ import Redis, { ClientType } from "../../Infrastructure/Redis";
 import Dictionary from "../../../Types/Dictionary";
 
 /*
- * Where the previous counter sample is kept so a later run can turn Redis'
- * monotonic counters into "did this happen since we last looked".
+ * What a single Redis/Valkey INFO call reports, for the admin health API
+ * (packages/App/API/AdminHealth.ts).
  *
- * Redis stores its own history here on purpose: the counters and the sample
- * then share a lifetime, so a Redis restart clears both together and cannot
- * leave a stale baseline that fabricates an enormous delta. The uptime guard
- * below covers the case where the sample somehow outlives a restart anyway.
- *
- * Under memory pressure with an eviction policy this key can itself be evicted,
- * which costs one interval of eviction detection: the next run finds no sample,
- * re-seeds it, and reports deltas again from there.
- */
-const COUNTER_SAMPLE_KEY: string = "oneuptime-instance-health-redis-sample";
-const COUNTER_SAMPLE_TTL_IN_SECONDS: number = 60 * 60;
-
-/*
- * The minimum age of the baseline the counter deltas are measured against, so
- * every evaluation answers "did this happen in at least the last half hour"
- * rather than "did it happen in the last five minutes".
- *
- * That span is what keeps an edge-triggered signal from flapping. Evictions
- * arriving in bursts would otherwise clear on the first quiet tick and fire
- * again on the next, emailing every other cycle.
- *
- * Two samples are kept rather than one because a single rolling baseline is a
- * TUMBLING window, not a sliding one: the tick immediately after a roll would
- * compare against a five-minute-old sample and a lone quiet tick there would
- * resolve the notification, re-notifying on the next burst of the same ongoing
- * incident. Measuring against the OLDER of two samples makes the lookback
- * always at least this long (and at most twice it).
+ * The counter history that turns INFO's cumulative totals into "did this
+ * happen recently" belongs to the Enterprise instance-health worker and lives
+ * next to it in ee/Server/Workers/InstanceHealth/RedisHealth.ts, which reads
+ * INFO through readInfoSnapshot below.
  */
-export const COUNTER_WINDOW_IN_SECONDS: number = 30 * 60;
 
 // Everything a single INFO call reports, with no history involved.
 export interface RedisInfoSnapshot {
@@ -56,36 +32,6 @@ export interface RedisInfoSnapshot {
   uptimeInSeconds: number;
 }
 
-export interface RedisHealthSnapshot extends RedisInfoSnapshot {
-  /*
-   * Increase over the baseline. null when this run has no baseline it can trust
-   * — no stored sample, or Redis restarted and reset the counters.
-   */
-  evictedKeysDelta: number | null;
-  rejectedConnectionsDelta: number | null;
-  /*
-   * How far back the deltas above actually reach, so messages can state the
-   * real span instead of quoting a nominal constant. null alongside null deltas.
-   */
-  counterWindowInSeconds: number | null;
-}
-
-export interface RedisCounterSample {
-  evictedKeys: number;
-  rejectedConnections: number;
-  uptimeInSeconds: number;
-}
-
-/*
- * The two retained samples. Deltas are measured against `older`, which keeps
- * the lookback at least COUNTER_WINDOW_IN_SECONDS at every tick; `newer` is the
- * one promoted to `older` at the next roll.
- */
-export interface RedisCounterSamples {
-  older: RedisCounterSample;
-  newer: RedisCounterSample;
-}
-
 /*
  * INFO is a flat `field:value` document split into `# Section` blocks. Section
  * headers and comment lines are dropped; field names are unique across sections
@@ -191,144 +137,11 @@ async function getMaxClients(
   return null;
 }
 
-function parseCounterSample(value: unknown): RedisCounterSample | null {
-  if (!value || typeof value !== "object" || Array.isArray(value)) {
-    return null;
-  }
-
-  const sample: Record<string, unknown> = value as Record<string, unknown>;
-  const evictedKeys: number = Number(sample["evictedKeys"]);
-  const rejectedConnections: number = Number(sample["rejectedConnections"]);
-  const uptimeInSeconds: number = Number(sample["uptimeInSeconds"]);
-
-  if (
-    !Number.isFinite(evictedKeys) ||
-    !Number.isFinite(rejectedConnections) ||
-    !Number.isFinite(uptimeInSeconds)
-  ) {
-    return null;
-  }
-
-  return { evictedKeys, rejectedConnections, uptimeInSeconds };
-}
-
-async function readCounterSamples(
-  client: ClientType,
-): Promise<RedisCounterSamples | null> {
-  try {
-    const raw: string | null = await client.get(COUNTER_SAMPLE_KEY);
-
-    if (!raw) {
-      return null;
-    }
-
-    const parsed: unknown = JSON.parse(raw);
-
-    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
-      return null;
-    }
-
-    const stored: Record<string, unknown> = parsed as Record<string, unknown>;
-    const older: RedisCounterSample | null = parseCounterSample(
-      stored["older"],
-    );
-    const newer: RedisCounterSample | null = parseCounterSample(
-      stored["newer"],
-    );
-
-    if (older && newer) {
-      return { older, newer };
-    }
-
-    /*
-     * A single-sample payload is what the previous release wrote. Treat it as
-     * both ends of the pair so an upgrade keeps its baseline instead of going
-     * blind for a window.
-     */
-    const legacy: RedisCounterSample | null = parseCounterSample(parsed);
-
-    return legacy ? { older: legacy, newer: legacy } : null;
-  } catch {
-    // A missing or unreadable sample simply means no delta this run.
-    return null;
-  }
-}
-
 /*
- * Returns whether the baseline was actually stored. The caller needs to know,
- * because a write can fail for a reason that does not stop the key from being
- * kept alive by other means — see getRedisHealthSnapshot.
+ * One INFO read, plus the CONFIG fallback for maxclients. Exported so the
+ * Enterprise worker's snapshot reads Redis exactly the way this file does.
  */
-async function writeCounterSamples(
-  client: ClientType,
-  samples: RedisCounterSamples,
-): Promise<boolean> {
-  try {
-    await client.set(
-      COUNTER_SAMPLE_KEY,
-      JSON.stringify(samples),
-      "EX",
-      COUNTER_SAMPLE_TTL_IN_SECONDS,
-    );
-    return true;
-  } catch {
-    /*
-     * Losing the baseline must never fail the health evaluation that produced
-     * this snapshot, so the error is swallowed and reported through the return.
-     */
-    return false;
-  }
-}
-
-// Keep a baseline alive across the runs that do not rewrite it.
-async function refreshCounterSampleTtl(client: ClientType): Promise<void> {
-  try {
-    await client.expire(COUNTER_SAMPLE_KEY, COUNTER_SAMPLE_TTL_IN_SECONDS);
-  } catch {
-    // Same reasoning as writeCounterSamples: never fail the evaluation over this.
-  }
-}
-
-/*
- * The pair rolls only once the NEWER sample has aged past the window, which
- * promotes it to `older` and starts a fresh one. Deltas are always measured
- * against `older`, so the lookback never drops below the window. Redis' own
- * uptime measures that age, which avoids trusting either machine's clock.
- */
-export function shouldRollCounterSamples(data: {
-  previous: RedisCounterSamples | null;
-  currentUptimeInSeconds: number;
-  didRestart: boolean;
-}): boolean {
-  if (!data.previous || data.didRestart) {
-    return true;
-  }
-
-  return (
-    data.currentUptimeInSeconds - data.previous.newer.uptimeInSeconds >=
-    COUNTER_WINDOW_IN_SECONDS
-  );
-}
-
-/*
- * Compare a monotonic counter against its previous value. A decrease, or a
- * shorter uptime than the sample was taken at, means Redis restarted and the
- * counter began again from zero — the difference is meaningless, so report no
- * delta rather than a fabricated spike.
- */
-export function getCounterDelta(data: {
-  current: number;
-  previous: number;
-  didRestart: boolean;
-}): number | null {
-  if (data.didRestart || data.current < data.previous) {
-    return null;
-  }
-
-  return data.current - data.previous;
-}
-
-async function readInfoSnapshot(
+export async function readInfoSnapshot(
   client: ClientType,
 ): Promise<RedisInfoSnapshot> {
   const info: Dictionary<string> = parseRedisInfo(await client.info());
@@ -341,9 +154,9 @@ async function readInfoSnapshot(
 
 /*
  * Read-only view for the admin health API. It deliberately does NOT touch the
- * stored counter sample: advancing that baseline here would consume the delta
- * the worker needs, so a page refresh could hide an eviction burst from the
- * notification that exists to catch it.
+ * counter sample the Enterprise health worker keeps: advancing that baseline
+ * here would consume the delta the worker needs, so a page refresh could hide
+ * an eviction burst from the notification that exists to catch it.
  */
 export async function getRedisInfoSnapshot(): Promise<RedisInfoSnapshot | null> {
   const client: ClientType | null = Redis.getClient();
@@ -354,81 +167,3 @@ export async function getRedisInfoSnapshot(): Promise<RedisInfoSnapshot | null>
 
   return await readInfoSnapshot(client);
 }
-
-/*
- * The evaluating worker's view: the same INFO read, plus the counter deltas
- * that turn cumulative totals into "did this happen in roughly the last
- * COUNTER_WINDOW_IN_SECONDS". Returns null when Redis is unreachable, which the
- * caller treats as "cannot evaluate" rather than "healthy".
- */
-export async function getRedisHealthSnapshot(): Promise<RedisHealthSnapshot | null> {
-  const client: ClientType | null = Redis.getClient();
-
-  if (!client || !Redis.isConnected()) {
-    return null;
-  }
-
-  const snapshot: RedisInfoSnapshot = await readInfoSnapshot(client);
-
-  const previousSamples: RedisCounterSamples | null =
-    await readCounterSamples(client);
-  const didRestart: boolean = Boolean(
-    previousSamples &&
-      previousSamples.older.uptimeInSeconds > snapshot.uptimeInSeconds,
-  );
-  // Deltas always measure against the older half, never the freshly rolled one.
-  const baseline: RedisCounterSample | null = previousSamples
-    ? previousSamples.older
-    : null;
-
-  const current: RedisCounterSample = {
-    evictedKeys: snapshot.evictedKeys,
-    rejectedConnections: snapshot.rejectedConnections,
-    uptimeInSeconds: snapshot.uptimeInSeconds,
-  };
-  const wantsRoll: boolean = shouldRollCounterSamples({
-    previous: previousSamples,
-    currentUptimeInSeconds: snapshot.uptimeInSeconds,
-    didRestart,
-  });
-  const didRoll: boolean = wantsRoll
-    ? await writeCounterSamples(client, {
-        older: previousSamples && !didRestart ? previousSamples.newer : current,
-        newer: current,
-      })
-    : false;
-
-  /*
-   * Keep the existing samples alive whenever they were not rolled — including
-   * when the roll FAILED, which is the case that matters most. Under the
-   * noeviction policy Redis rejects SET once it is out of memory but still
-   * serves EXPIRE, so letting a failed write skip the refresh would expire the
-   * baseline precisely when memory is full: eviction detection would then go
-   * permanently blind at the moment it is needed, because every later write
-   * fails the same way and the key never returns.
-   */
-  if (!didRoll) {
-    await refreshCounterSampleTtl(client);
-  }
-
-  return {
-    ...snapshot,
-    counterWindowInSeconds: baseline
-      ? snapshot.uptimeInSeconds - baseline.uptimeInSeconds
-      : null,
-    evictedKeysDelta: baseline
-      ? getCounterDelta({
-          current: snapshot.evictedKeys,
-          previous: baseline.evictedKeys,
-          didRestart,
-        })
-      : null,
-    rejectedConnectionsDelta: baseline
-      ? getCounterDelta({
-          current: snapshot.rejectedConnections,
-          previous: baseline.rejectedConnections,
-          didRestart,
-        })
-      : null,
-  };
-}

package/Server/Utils/Monitor/AffectedResourceList.ts

@@ -0,0 +1,148 @@
+import { escapeMarkdownInline } from "../../../Utils/Markdown/MarkdownEscape";
+import RootCauseList, {
+  RootCauseListDetail,
+  RootCauseListItem,
+} from "./RootCauseList";
+
+/*
+ * The "Affected Resources" block of a platform monitor's root cause.
+ *
+ * Kubernetes, Proxmox, VMware, Docker Swarm and Ceph monitors each attach a
+ * per-resource breakdown to the incident / alert they open, and each one
+ * used to render it as a GitHub-flavoured table: up to six columns of long,
+ * unbreakable identifiers (pod names, node names, pool paths). That table
+ * is read in three places, and it read badly in all of them:
+ *
+ *   - the email: the DetailBox card leaves roughly 416px for the root
+ *     cause, so every cell wrapped mid-identifier and a ten-row table
+ *     became a wall of fragments,
+ *   - Slack: mrkdwn has no tables, so the incident feed posted the raw
+ *     "| … |" pipe rows, and convertMarkdownToSlackRichText unrolled each
+ *     row into a "_Row 1_" block of "*Header:* value" lines,
+ *   - the dashboard: the widest column decided the layout, and a single
+ *     long node name squeezed the rest into two-line cells.
+ *
+ * So the breakdown is a ranked list instead. Each resource is one numbered
+ * item — worst first, the same order the table used — that says what the
+ * resource is, names it, and gives its value:
+ *
+ *     1. **Pod** `checkout-7d9f` — **3**
+ *        - Namespace: `payments`
+ *        - Deployment: `checkout`
+ *        - Node: `gke-prod-pool-1-abcd`
+ *
+ * and the rest of its identity sits underneath as labelled bullets. A
+ * missing attribute is simply left out, where the table had to print "-".
+ *
+ * The list itself is a RootCauseList, the shape the Breaching Samples block
+ * of a metric monitor's root cause uses too.
+ */
+
+export interface AffectedResourceListDetail {
+  // Plain text. Escaped when rendered.
+  label: string;
+  // Markdown, rendered as-is. An empty value drops the whole detail.
+  value: string;
+}
+
+export interface AffectedResourceListEntry {
+  // Plain text, e.g. "Pod" or "Virtual Machine". Escaped when rendered.
+  kind: string;
+  // Markdown, normally built with AffectedResourceList.code().
+  name: string;
+  // Markdown — the formatted metric value.
+  value: string;
+  details: Array<AffectedResourceListDetail>;
+}
+
+export default class AffectedResourceList {
+  // How many resources the list shows before it summarises the rest.
+  public static readonly MAX_ENTRIES: number = 10;
+
+  public static render(input: {
+    // Plain text, e.g. "Affected Resources".
+    heading: string;
+    // Plain text, e.g. "affected resources" in "... and 3 more affected resources".
+    overflowNoun: string;
+    // Every resource that qualified for the list, including those not shown.
+    totalCount: number;
+    // The resources to show, already sorted worst first.
+    entries: Array<AffectedResourceListEntry>;
+  }): string {
+    const items: Array<RootCauseListItem> = input.entries.map(
+      (entry: AffectedResourceListEntry): RootCauseListItem => {
+        /*
+         * The kind is bold so each item's first line stands out from the
+         * detail bullets under it — the value at the end of the line is
+         * bold too, so the eye lands on what it is and how bad it is.
+         */
+        const kind: string = escapeMarkdownInline(entry.kind).trim();
+
+        return {
+          title: [kind ? `**${kind}**` : "", entry.name.trim()]
+            .filter((part: string): boolean => {
+              return part.length > 0;
+            })
+            .join(" "),
+          value: entry.value,
+          details: entry.details.map(
+            (detail: AffectedResourceListDetail): RootCauseListDetail => {
+              return {
+                label: escapeMarkdownInline(detail.label),
+                value: detail.value,
+              };
+            },
+          ),
+        };
+      },
+    );
+
+    const lines: Array<string> = [RootCauseList.render(items)];
+
+    const hiddenCount: number = input.totalCount - input.entries.length;
+
+    if (hiddenCount > 0) {
+      /*
+       * The blank line matters: without it the summary is a lazy
+       * continuation of the last item's final bullet, not a paragraph of
+       * its own.
+       */
+      lines.push("");
+      lines.push(
+        `*... and ${hiddenCount} more ${escapeMarkdownInline(input.overflowNoun)}*`,
+      );
+    }
+
+    return `\n\n**${escapeMarkdownInline(input.heading)}** (${input.totalCount} total)\n\n${lines.join("\n")}`;
+  }
+
+  /*
+   * A resource identifier as an inline code span. The identifiers come from
+   * telemetry attributes, and a VMware VM or a Proxmox guest can be named
+   * anything — RootCauseList.code keeps a name with backticks or line breaks
+   * inside its span.
+   */
+  public static code(value: string | undefined | null): string {
+    return RootCauseList.code(value);
+  }
+
+  /*
+   * A display name alongside the identifier it abbreviates — a Proxmox
+   * guest's name and its `qemu/100` id, a Ceph pool's name and number, a
+   * vSphere resource pool's name and inventory path. Either half may be
+   * missing; when both are the same only one is shown.
+   */
+  public static codeWithId(input: {
+    name?: string | undefined;
+    id?: string | undefined;
+  }): string {
+    const name: string = AffectedResourceList.code(input.name);
+    const id: string = AffectedResourceList.code(input.id);
+
+    if (name && id && name !== id) {
+      return `${name} (${id})`;
+    }
+
+    return name || id;
+  }
+}

package/Server/Utils/Monitor/Criteria/CompareCriteria.ts

@@ -984,7 +984,7 @@ export default class CompareCriteria {
        * reads naturally ("is 42, 55, 60"). For larger arrays, summarize
        * — dumping 30+ numbers on one line makes the root cause unreadable
        * and the detailed breakdown is shown in the Breaching Samples
-       * table below.
+       * list below.
        */
       const MAX_INLINE: number = 5;
 

package/Server/Utils/Monitor/Criteria/EvaluateOverTime.ts

@@ -17,6 +17,7 @@ import MetricService from "../../../Services/MetricService";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 import InBetween from "../../../../Types/BaseDatabase/InBetween";
 import CronTab from "../../CronTab";
+import MonitoringIntervalUtil from "../../../../Utils/Monitor/MonitoringIntervalUtil";
 import logger from "../../Logger";
 
 /**
@@ -488,8 +489,16 @@ export default class EvaluateOverTime {
     monitoringInterval?: string | undefined;
     sortedSampleTimes: Array<number>;
   }): number | null {
+    /*
+     * Through the normalizer first: a monitor storing "5m" has a perfectly
+     * well-defined cadence, and treating it as "cadence unknown" made
+     * over-time windows fall back to guessing from the data.
+     */
     const configuredIntervalInSeconds: number | null =
-      CronTab.getIntervalInSeconds(data.monitoringInterval);
+      CronTab.getIntervalInSeconds(
+        MonitoringIntervalUtil.toCronOrNull(data.monitoringInterval) ||
+          data.monitoringInterval,
+      );
 
     if (configuredIntervalInSeconds !== null) {
       return configuredIntervalInSeconds;

package/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.ts

@@ -1004,7 +1004,7 @@ export default class MetricMonitorCriteria {
     const sigmaAbs: number = Math.abs(firstBreach.sigma);
 
     /*
-     * Same rendering as the breaching-samples table and the comparison
+     * Same rendering as the breaching-samples list and the comparison
      * sentence — an anomaly on a memory metric reads "1.07 GB", not
      * "1073741824.000 By". σ is a count of standard deviations, not a
      * quantity in the metric's unit, so it stays a bare number; the