@oneuptime/common 13.0.7 → 14.0.0

package/Server/Utils/InstanceHealth/RedisHealth.ts

@@ -2,38 +2,14 @@ import Redis, { ClientType } from "../../Infrastructure/Redis";
 import Dictionary from "../../../Types/Dictionary";
 
 /*
- * Where the previous counter sample is kept so a later run can turn Redis'
- * monotonic counters into "did this happen since we last looked".
+ * What a single Redis/Valkey INFO call reports, for the admin health API
+ * (packages/App/API/AdminHealth.ts).
  *
- * Redis stores its own history here on purpose: the counters and the sample
- * then share a lifetime, so a Redis restart clears both together and cannot
- * leave a stale baseline that fabricates an enormous delta. The uptime guard
- * below covers the case where the sample somehow outlives a restart anyway.
- *
- * Under memory pressure with an eviction policy this key can itself be evicted,
- * which costs one interval of eviction detection: the next run finds no sample,
- * re-seeds it, and reports deltas again from there.
- */
-const COUNTER_SAMPLE_KEY: string = "oneuptime-instance-health-redis-sample";
-const COUNTER_SAMPLE_TTL_IN_SECONDS: number = 60 * 60;
-
-/*
- * The minimum age of the baseline the counter deltas are measured against, so
- * every evaluation answers "did this happen in at least the last half hour"
- * rather than "did it happen in the last five minutes".
- *
- * That span is what keeps an edge-triggered signal from flapping. Evictions
- * arriving in bursts would otherwise clear on the first quiet tick and fire
- * again on the next, emailing every other cycle.
- *
- * Two samples are kept rather than one because a single rolling baseline is a
- * TUMBLING window, not a sliding one: the tick immediately after a roll would
- * compare against a five-minute-old sample and a lone quiet tick there would
- * resolve the notification, re-notifying on the next burst of the same ongoing
- * incident. Measuring against the OLDER of two samples makes the lookback
- * always at least this long (and at most twice it).
+ * The counter history that turns INFO's cumulative totals into "did this
+ * happen recently" belongs to the Enterprise instance-health worker and lives
+ * next to it in ee/Server/Workers/InstanceHealth/RedisHealth.ts, which reads
+ * INFO through readInfoSnapshot below.
  */
-export const COUNTER_WINDOW_IN_SECONDS: number = 30 * 60;
 
 // Everything a single INFO call reports, with no history involved.
 export interface RedisInfoSnapshot {
@@ -56,36 +32,6 @@ export interface RedisInfoSnapshot {
   uptimeInSeconds: number;
 }
 
-export interface RedisHealthSnapshot extends RedisInfoSnapshot {
-  /*
-   * Increase over the baseline. null when this run has no baseline it can trust
-   * — no stored sample, or Redis restarted and reset the counters.
-   */
-  evictedKeysDelta: number | null;
-  rejectedConnectionsDelta: number | null;
-  /*
-   * How far back the deltas above actually reach, so messages can state the
-   * real span instead of quoting a nominal constant. null alongside null deltas.
-   */
-  counterWindowInSeconds: number | null;
-}
-
-export interface RedisCounterSample {
-  evictedKeys: number;
-  rejectedConnections: number;
-  uptimeInSeconds: number;
-}
-
-/*
- * The two retained samples. Deltas are measured against `older`, which keeps
- * the lookback at least COUNTER_WINDOW_IN_SECONDS at every tick; `newer` is the
- * one promoted to `older` at the next roll.
- */
-export interface RedisCounterSamples {
-  older: RedisCounterSample;
-  newer: RedisCounterSample;
-}
-
 /*
  * INFO is a flat `field:value` document split into `# Section` blocks. Section
  * headers and comment lines are dropped; field names are unique across sections
@@ -191,144 +137,11 @@ async function getMaxClients(
   return null;
 }
 
-function parseCounterSample(value: unknown): RedisCounterSample | null {
-  if (!value || typeof value !== "object" || Array.isArray(value)) {
-    return null;
-  }
-
-  const sample: Record<string, unknown> = value as Record<string, unknown>;
-  const evictedKeys: number = Number(sample["evictedKeys"]);
-  const rejectedConnections: number = Number(sample["rejectedConnections"]);
-  const uptimeInSeconds: number = Number(sample["uptimeInSeconds"]);
-
-  if (
-    !Number.isFinite(evictedKeys) ||
-    !Number.isFinite(rejectedConnections) ||
-    !Number.isFinite(uptimeInSeconds)
-  ) {
-    return null;
-  }
-
-  return { evictedKeys, rejectedConnections, uptimeInSeconds };
-}
-
-async function readCounterSamples(
-  client: ClientType,
-): Promise<RedisCounterSamples | null> {
-  try {
-    const raw: string | null = await client.get(COUNTER_SAMPLE_KEY);
-
-    if (!raw) {
-      return null;
-    }
-
-    const parsed: unknown = JSON.parse(raw);
-
-    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
-      return null;
-    }
-
-    const stored: Record<string, unknown> = parsed as Record<string, unknown>;
-    const older: RedisCounterSample | null = parseCounterSample(
-      stored["older"],
-    );
-    const newer: RedisCounterSample | null = parseCounterSample(
-      stored["newer"],
-    );
-
-    if (older && newer) {
-      return { older, newer };
-    }
-
-    /*
-     * A single-sample payload is what the previous release wrote. Treat it as
-     * both ends of the pair so an upgrade keeps its baseline instead of going
-     * blind for a window.
-     */
-    const legacy: RedisCounterSample | null = parseCounterSample(parsed);
-
-    return legacy ? { older: legacy, newer: legacy } : null;
-  } catch {
-    // A missing or unreadable sample simply means no delta this run.
-    return null;
-  }
-}
-
 /*
- * Returns whether the baseline was actually stored. The caller needs to know,
- * because a write can fail for a reason that does not stop the key from being
- * kept alive by other means — see getRedisHealthSnapshot.
+ * One INFO read, plus the CONFIG fallback for maxclients. Exported so the
+ * Enterprise worker's snapshot reads Redis exactly the way this file does.
  */
-async function writeCounterSamples(
-  client: ClientType,
-  samples: RedisCounterSamples,
-): Promise<boolean> {
-  try {
-    await client.set(
-      COUNTER_SAMPLE_KEY,
-      JSON.stringify(samples),
-      "EX",
-      COUNTER_SAMPLE_TTL_IN_SECONDS,
-    );
-    return true;
-  } catch {
-    /*
-     * Losing the baseline must never fail the health evaluation that produced
-     * this snapshot, so the error is swallowed and reported through the return.
-     */
-    return false;
-  }
-}
-
-// Keep a baseline alive across the runs that do not rewrite it.
-async function refreshCounterSampleTtl(client: ClientType): Promise<void> {
-  try {
-    await client.expire(COUNTER_SAMPLE_KEY, COUNTER_SAMPLE_TTL_IN_SECONDS);
-  } catch {
-    // Same reasoning as writeCounterSamples: never fail the evaluation over this.
-  }
-}
-
-/*
- * The pair rolls only once the NEWER sample has aged past the window, which
- * promotes it to `older` and starts a fresh one. Deltas are always measured
- * against `older`, so the lookback never drops below the window. Redis' own
- * uptime measures that age, which avoids trusting either machine's clock.
- */
-export function shouldRollCounterSamples(data: {
-  previous: RedisCounterSamples | null;
-  currentUptimeInSeconds: number;
-  didRestart: boolean;
-}): boolean {
-  if (!data.previous || data.didRestart) {
-    return true;
-  }
-
-  return (
-    data.currentUptimeInSeconds - data.previous.newer.uptimeInSeconds >=
-    COUNTER_WINDOW_IN_SECONDS
-  );
-}
-
-/*
- * Compare a monotonic counter against its previous value. A decrease, or a
- * shorter uptime than the sample was taken at, means Redis restarted and the
- * counter began again from zero — the difference is meaningless, so report no
- * delta rather than a fabricated spike.
- */
-export function getCounterDelta(data: {
-  current: number;
-  previous: number;
-  didRestart: boolean;
-}): number | null {
-  if (data.didRestart || data.current < data.previous) {
-    return null;
-  }
-
-  return data.current - data.previous;
-}
-
-async function readInfoSnapshot(
+export async function readInfoSnapshot(
   client: ClientType,
 ): Promise<RedisInfoSnapshot> {
   const info: Dictionary<string> = parseRedisInfo(await client.info());
@@ -341,9 +154,9 @@ async function readInfoSnapshot(
 
 /*
  * Read-only view for the admin health API. It deliberately does NOT touch the
- * stored counter sample: advancing that baseline here would consume the delta
- * the worker needs, so a page refresh could hide an eviction burst from the
- * notification that exists to catch it.
+ * counter sample the Enterprise health worker keeps: advancing that baseline
+ * here would consume the delta the worker needs, so a page refresh could hide
+ * an eviction burst from the notification that exists to catch it.
  */
 export async function getRedisInfoSnapshot(): Promise<RedisInfoSnapshot | null> {
   const client: ClientType | null = Redis.getClient();
@@ -354,81 +167,3 @@ export async function getRedisInfoSnapshot(): Promise<RedisInfoSnapshot | null>
 
   return await readInfoSnapshot(client);
 }
-
-/*
- * The evaluating worker's view: the same INFO read, plus the counter deltas
- * that turn cumulative totals into "did this happen in roughly the last
- * COUNTER_WINDOW_IN_SECONDS". Returns null when Redis is unreachable, which the
- * caller treats as "cannot evaluate" rather than "healthy".
- */
-export async function getRedisHealthSnapshot(): Promise<RedisHealthSnapshot | null> {
-  const client: ClientType | null = Redis.getClient();
-
-  if (!client || !Redis.isConnected()) {
-    return null;
-  }
-
-  const snapshot: RedisInfoSnapshot = await readInfoSnapshot(client);
-
-  const previousSamples: RedisCounterSamples | null =
-    await readCounterSamples(client);
-  const didRestart: boolean = Boolean(
-    previousSamples &&
-      previousSamples.older.uptimeInSeconds > snapshot.uptimeInSeconds,
-  );
-  // Deltas always measure against the older half, never the freshly rolled one.
-  const baseline: RedisCounterSample | null = previousSamples
-    ? previousSamples.older
-    : null;
-
-  const current: RedisCounterSample = {
-    evictedKeys: snapshot.evictedKeys,
-    rejectedConnections: snapshot.rejectedConnections,
-    uptimeInSeconds: snapshot.uptimeInSeconds,
-  };
-  const wantsRoll: boolean = shouldRollCounterSamples({
-    previous: previousSamples,
-    currentUptimeInSeconds: snapshot.uptimeInSeconds,
-    didRestart,
-  });
-  const didRoll: boolean = wantsRoll
-    ? await writeCounterSamples(client, {
-        older: previousSamples && !didRestart ? previousSamples.newer : current,
-        newer: current,
-      })
-    : false;
-
-  /*
-   * Keep the existing samples alive whenever they were not rolled — including
-   * when the roll FAILED, which is the case that matters most. Under the
-   * noeviction policy Redis rejects SET once it is out of memory but still
-   * serves EXPIRE, so letting a failed write skip the refresh would expire the
-   * baseline precisely when memory is full: eviction detection would then go
-   * permanently blind at the moment it is needed, because every later write
-   * fails the same way and the key never returns.
-   */
-  if (!didRoll) {
-    await refreshCounterSampleTtl(client);
-  }
-
-  return {
-    ...snapshot,
-    counterWindowInSeconds: baseline
-      ? snapshot.uptimeInSeconds - baseline.uptimeInSeconds
-      : null,
-    evictedKeysDelta: baseline
-      ? getCounterDelta({
-          current: snapshot.evictedKeys,
-          previous: baseline.evictedKeys,
-          didRestart,
-        })
-      : null,
-    rejectedConnectionsDelta: baseline
-      ? getCounterDelta({
-          current: snapshot.rejectedConnections,
-          previous: baseline.rejectedConnections,
-          didRestart,
-        })
-      : null,
-  };
-}

package/Server/Utils/MasterAdminPageAccess.ts

@@ -0,0 +1,61 @@
+import UserMiddleware from "../Middleware/UserAuthorization";
+import { ExpressRequest, ExpressResponse } from "./Express";
+import JSONWebToken from "./JsonWebToken";
+import logger from "./Logger";
+import Response from "./Response";
+import NotAuthorizedException from "../../Types/Exception/NotAuthorizedException";
+import { JSONObject } from "../../Types/JSON";
+import JSONWebTokenData from "../../Types/JsonWebTokenData";
+
+export const NOT_A_MASTER_ADMIN_MESSAGE: string =
+  "Unauthorized: Only master admins can access the admin dashboard.";
+
+/*
+ * Gate for the Admin Dashboard's index page (the HTML shell, not its APIs).
+ *
+ * It refuses only a caller it can positively identify as NOT a master admin.
+ * A missing access token, or one that no longer decodes, is served the shell:
+ * the access-token cookie expires with the JWT inside it, so an admin who
+ * reloads a tab left open past the token lifetime arrives here with no token
+ * at all, and answering that with a 422 JSON page meant the app never loaded
+ * and so never got to refresh the session. The shell holds nothing the public
+ * static bundle does not; once it loads, the SPA sends a signed-out visitor to
+ * the login page, and every admin API it calls refreshes an expired session
+ * (or answers 401) on its own.
+ */
+export const ensureMasterAdminPageAccess: (data: {
+  req: ExpressRequest;
+  res: ExpressResponse;
+  service: string;
+}) => Promise<JSONObject> = async (data: {
+  req: ExpressRequest;
+  res: ExpressResponse;
+  service: string;
+}): Promise<JSONObject> => {
+  const accessToken: string | undefined =
+    UserMiddleware.getAccessTokenFromExpressRequest(data.req);
+
+  if (!accessToken) {
+    return {};
+  }
+
+  let authData: JSONWebTokenData;
+
+  try {
+    authData = JSONWebToken.decode(accessToken);
+  } catch (error) {
+    // Expired or unreadable: let the app load and refresh it.
+    logger.debug(error, { service: data.service });
+    return {};
+  }
+
+  if (!authData.isMasterAdmin) {
+    Response.sendErrorResponse(
+      data.req,
+      data.res,
+      new NotAuthorizedException(NOT_A_MASTER_ADMIN_MESSAGE),
+    );
+  }
+
+  return {};
+};

package/Server/Utils/Monitor/AffectedResourceList.ts

@@ -0,0 +1,190 @@
+import { escapeMarkdownInline } from "../../../Utils/Markdown/MarkdownEscape";
+
+/*
+ * The "Affected Resources" block of a platform monitor's root cause.
+ *
+ * Kubernetes, Proxmox, VMware, Docker Swarm and Ceph monitors each attach a
+ * per-resource breakdown to the incident / alert they open, and each one
+ * used to render it as a GitHub-flavoured table: up to six columns of long,
+ * unbreakable identifiers (pod names, node names, pool paths). That table
+ * is read in three places, and it read badly in all of them:
+ *
+ *   - the email: the DetailBox card leaves roughly 416px for the root
+ *     cause, so every cell wrapped mid-identifier and a ten-row table
+ *     became a wall of fragments,
+ *   - Slack: mrkdwn has no tables, so the incident feed posted the raw
+ *     "| … |" pipe rows, and convertMarkdownToSlackRichText unrolled each
+ *     row into a "_Row 1_" block of "*Header:* value" lines,
+ *   - the dashboard: the widest column decided the layout, and a single
+ *     long node name squeezed the rest into two-line cells.
+ *
+ * So the breakdown is a ranked list instead. Each resource is one numbered
+ * item — worst first, the same order the table used — that says what the
+ * resource is, names it, and gives its value:
+ *
+ *     1. **Pod** `checkout-7d9f` — **3**
+ *        - Namespace: `payments`
+ *        - Deployment: `checkout`
+ *        - Node: `gke-prod-pool-1-abcd`
+ *
+ * and the rest of its identity sits underneath as labelled bullets. A
+ * missing attribute is simply left out, where the table had to print "-".
+ *
+ * Plain markdown, deliberately: marked (email), react-markdown (dashboard),
+ * slackify-markdown (Slack), the dashboard editor's WYSIWYG converters and
+ * Markdown.convertToPlainText all understand a nested list, so nothing
+ * downstream needs to know this block exists.
+ */
+
+export interface AffectedResourceListDetail {
+  // Plain text. Escaped when rendered.
+  label: string;
+  // Markdown, rendered as-is. An empty value drops the whole detail.
+  value: string;
+}
+
+export interface AffectedResourceListEntry {
+  // Plain text, e.g. "Pod" or "Virtual Machine". Escaped when rendered.
+  kind: string;
+  // Markdown, normally built with AffectedResourceList.code().
+  name: string;
+  // Markdown — the formatted metric value.
+  value: string;
+  details: Array<AffectedResourceListDetail>;
+}
+
+export default class AffectedResourceList {
+  // How many resources the list shows before it summarises the rest.
+  public static readonly MAX_ENTRIES: number = 10;
+
+  public static render(input: {
+    // Plain text, e.g. "Affected Resources".
+    heading: string;
+    // Plain text, e.g. "affected resources" in "... and 3 more affected resources".
+    overflowNoun: string;
+    // Every resource that qualified for the list, including those not shown.
+    totalCount: number;
+    // The resources to show, already sorted worst first.
+    entries: Array<AffectedResourceListEntry>;
+  }): string {
+    const lines: Array<string> = [];
+
+    input.entries.forEach(
+      (entry: AffectedResourceListEntry, index: number): void => {
+        const marker: string = `${index + 1}.`;
+
+        /*
+         * A nested bullet belongs to its item only when it is indented to
+         * the item's content column, which is one past the marker — three
+         * spaces under "1.", four under "10.". A fixed three-space indent
+         * would push item 10's details out into a separate top-level list.
+         */
+        const indent: string = " ".repeat(marker.length + 1);
+
+        /*
+         * The kind is bold so each item's first line stands out from the
+         * detail bullets under it — the value at the end of the line is
+         * bold too, so the eye lands on what it is and how bad it is.
+         */
+        const kind: string = escapeMarkdownInline(entry.kind).trim();
+        const title: string = [kind ? `**${kind}**` : "", entry.name.trim()]
+          .filter((part: string): boolean => {
+            return part.length > 0;
+          })
+          .join(" ");
+
+        lines.push(`${marker} ${title} — ${entry.value}`);
+
+        for (const detail of entry.details) {
+          if (!detail.value || detail.value.trim().length === 0) {
+            continue;
+          }
+
+          lines.push(
+            `${indent}- ${escapeMarkdownInline(detail.label)}: ${detail.value}`,
+          );
+        }
+      },
+    );
+
+    const hiddenCount: number = input.totalCount - input.entries.length;
+
+    if (hiddenCount > 0) {
+      /*
+       * The blank line matters: without it the summary is a lazy
+       * continuation of the last item's final bullet, not a paragraph of
+       * its own.
+       */
+      lines.push("");
+      lines.push(
+        `*... and ${hiddenCount} more ${escapeMarkdownInline(input.overflowNoun)}*`,
+      );
+    }
+
+    return `\n\n**${escapeMarkdownInline(input.heading)}** (${input.totalCount} total)\n\n${lines.join("\n")}`;
+  }
+
+  /*
+   * A resource identifier as an inline code span.
+   *
+   * The identifiers come from telemetry attributes, and a VMware VM or a
+   * Proxmox guest can be named anything. Wrapping a name that contains a
+   * backtick in single backticks closes the span early and spills the rest
+   * of the name — and whatever markdown it contains — into the list. So the
+   * fence is always one backtick longer than the longest run inside the
+   * value, and padded with a space on each side when the value has any
+   * backticks at all (CommonMark strips exactly one from each side, so the
+   * padding never shows). Line breaks become spaces: a newline inside a
+   * list item would end the item.
+   */
+  public static code(value: string | undefined | null): string {
+    if (value === undefined || value === null) {
+      return "";
+    }
+
+    const text: string = String(value)
+      .replace(/\s*[\r\n]+\s*/g, " ")
+      .trim();
+
+    if (text.length === 0) {
+      return "";
+    }
+
+    const backtickRuns: Array<string> = text.match(/`+/g) || [];
+
+    const longestRun: number = backtickRuns.reduce(
+      (longest: number, run: string): number => {
+        return Math.max(longest, run.length);
+      },
+      0,
+    );
+
+    if (longestRun === 0) {
+      return `\`${text}\``;
+    }
+
+    const fence: string = "`".repeat(longestRun + 1);
+
+    return `${fence} ${text} ${fence}`;
+  }
+
+  /*
+   * A display name alongside the identifier it abbreviates — a Proxmox
+   * guest's name and its `qemu/100` id, a Ceph pool's name and number, a
+   * vSphere resource pool's name and inventory path. Either half may be
+   * missing; when both are the same only one is shown.
+   */
+  public static codeWithId(input: {
+    name?: string | undefined;
+    id?: string | undefined;
+  }): string {
+    const name: string = AffectedResourceList.code(input.name);
+    const id: string = AffectedResourceList.code(input.id);
+
+    if (name && id && name !== id) {
+      return `${name} (${id})`;
+    }
+
+    return name || id;
+  }
+}

package/Server/Utils/Monitor/Criteria/EvaluateOverTime.ts

@@ -17,6 +17,7 @@ import MetricService from "../../../Services/MetricService";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 import InBetween from "../../../../Types/BaseDatabase/InBetween";
 import CronTab from "../../CronTab";
+import MonitoringIntervalUtil from "../../../../Utils/Monitor/MonitoringIntervalUtil";
 import logger from "../../Logger";
 
 /**
@@ -488,8 +489,16 @@ export default class EvaluateOverTime {
     monitoringInterval?: string | undefined;
     sortedSampleTimes: Array<number>;
   }): number | null {
+    /*
+     * Through the normalizer first: a monitor storing "5m" has a perfectly
+     * well-defined cadence, and treating it as "cadence unknown" made
+     * over-time windows fall back to guessing from the data.
+     */
     const configuredIntervalInSeconds: number | null =
-      CronTab.getIntervalInSeconds(data.monitoringInterval);
+      CronTab.getIntervalInSeconds(
+        MonitoringIntervalUtil.toCronOrNull(data.monitoringInterval) ||
+          data.monitoringInterval,
+      );
 
     if (configuredIntervalInSeconds !== null) {
       return configuredIntervalInSeconds;

package/Server/Utils/Monitor/MonitorAlert.ts

@@ -86,6 +86,7 @@ export default class MonitorAlert {
       select: {
         _id: true,
         title: true,
+        createdAt: true,
         createdCriteriaId: true,
         projectId: true,
         alertNumber: true,
@@ -514,6 +515,7 @@ export default class MonitorAlert {
               relatedAlertNumber: alreadyOpenAlert?.alertNumber,
               relatedAlertNumberWithPrefix:
                 alreadyOpenAlert?.alertNumberWithPrefix,
+              relatedAlertCreatedAt: alreadyOpenAlert?.createdAt,
               at: OneUptimeDate.getCurrentDate(),
             });
             continue;
@@ -794,7 +796,7 @@ export default class MonitorAlert {
             relatedAlertId: createdAlert.id?.toString(),
             relatedAlertNumber: createdAlert.alertNumber,
             relatedAlertNumberWithPrefix: createdAlert.alertNumberWithPrefix,
-            at: OneUptimeDate.getCurrentDate(),
+            at: createdAlert.createdAt || OneUptimeDate.getCurrentDate(),
           });
         } catch (err) {
           /*