@oneuptime/common 14.0.3 → 14.0.6

package/Utils/Telemetry/DatabaseServerEntityKeys.ts

@@ -0,0 +1,275 @@
+import { keyForDatabaseEndpoint, keyForDatabaseServerRow } from "./EntityKey";
+import {
+  DatabaseEndpoint,
+  parseDatabaseEndpointString,
+} from "../../Types/DatabaseServer/DatabaseEndpoint";
+
+/*
+ * The entity-key set of a DatabaseServer row — the ONE definition of "this
+ * database's telemetry" shared by the dashboard pages (Logs / Traces /
+ * Metrics / Overview), the explorer facet and anything else that scopes by
+ * database. A row's keys are:
+ *
+ *   - its ROW key (keyForDatabaseServerRow): ingest stamps it on every batch
+ *     that resolved to this row — by its `oneuptime.database.server.id`
+ *     link or by an endpoint it owns — so linked telemetry is this
+ *     database's whatever address (or none) the batch reported;
+ *   - one `database.server` key per stored endpoint (DatabaseServerEndpoint
+ *     rows): exactly the endpoints Settings shows, no derived twins, so what
+ *     the page queries is what the user can see and edit;
+ *   - its member keys: the Kubernetes pods / Deployments and Docker / Podman
+ *     containers it runs as, remembered with a last-seen time so a pod that
+ *     restarted (new name, new container id) keeps its crash logs visible.
+ *
+ * Kept out of EntityKey.ts on purpose: that module must stay a leaf, and
+ * this one needs the endpoint parser.
+ */
+
+/** entityKey → ISO timestamp it was last seen as a member. */
+export type DatabaseServerMemberKeys = Record<string, string>;
+
+export const DEFAULT_MEMBER_KEY_MAX_AGE_DAYS: number = 30;
+export const DEFAULT_MEMBER_KEY_MAX_COUNT: number = 200;
+export const MAX_DATABASE_SERVER_ENTITY_KEYS: number = 200;
+
+const ENTITY_KEY_REGEX: RegExp = /^[0-9a-f]{16}$/;
+const DAY_IN_MS: number = 24 * 60 * 60 * 1000;
+
+function normalizeEntityKey(value: unknown): string | null {
+  if (typeof value !== "string") {
+    return null;
+  }
+  const key: string = value.trim().toLowerCase();
+  return ENTITY_KEY_REGEX.test(key) ? key : null;
+}
+
+function toTimestamp(value: unknown): number | null {
+  if (typeof value !== "string" && !(value instanceof Date)) {
+    return null;
+  }
+  const time: number =
+    value instanceof Date ? value.getTime() : Date.parse(value);
+  return Number.isFinite(time) ? time : null;
+}
+
+/*
+ * Read a stored member-key value leniently: the canonical `{ key: iso }`
+ * map, or an array of `{ key, lastSeenAt }` entries (plus, when
+ * `acceptBareKeys`, bare key strings, treated as oldest). Anything malformed
+ * is skipped, never thrown on — the column is JSON written by a worker.
+ */
+function readMemberKeys(
+  value: unknown,
+  acceptBareKeys: boolean = false,
+): Map<string, number> {
+  const entries: Map<string, number> = new Map<string, number>();
+
+  const add: (rawKey: unknown, rawTime: unknown) => void = (
+    rawKey: unknown,
+    rawTime: unknown,
+  ): void => {
+    const key: string | null = normalizeEntityKey(rawKey);
+    const time: number | null = toTimestamp(rawTime);
+    if (key === null || time === null) {
+      return;
+    }
+    const existing: number | undefined = entries.get(key);
+    if (existing === undefined || time > existing) {
+      entries.set(key, time);
+    }
+  };
+
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      if (acceptBareKeys && typeof item === "string") {
+        add(item, new Date(0));
+      } else if (item && typeof item === "object") {
+        add(
+          (item as { key?: unknown }).key,
+          (item as { lastSeenAt?: unknown }).lastSeenAt,
+        );
+      }
+    }
+    return entries;
+  }
+
+  if (value && typeof value === "object") {
+    for (const rawKey of Object.keys(value as Record<string, unknown>)) {
+      add(rawKey, (value as Record<string, unknown>)[rawKey]);
+    }
+  }
+
+  return entries;
+}
+
+// Most recent first; ties broken by key so the order is stable.
+function sortByRecency(
+  entries: Map<string, number>,
+): Array<{ key: string; time: number }> {
+  return Array.from(entries.entries())
+    .map(([key, time]: [string, number]): { key: string; time: number } => {
+      return { key, time };
+    })
+    .sort(
+      (
+        a: { key: string; time: number },
+        b: { key: string; time: number },
+      ): number => {
+        if (b.time !== a.time) {
+          return b.time - a.time;
+        }
+        if (a.key < b.key) {
+          return -1;
+        }
+        return a.key > b.key ? 1 : 0;
+      },
+    );
+}
+
+/**
+ * Merge the member keys seen this run into the stored set: seen keys are
+ * stamped `now`, older ones are kept until they have not been seen for
+ * `maxAgeDays` (default 30 — a restarted pod's logs stay on the page), and
+ * the set is capped at `max` (default 200) by recency. Timestamps in the
+ * future are clamped to `now`. Never throws.
+ */
+export function mergeDatabaseServerMemberKeys(
+  existing: unknown,
+  seenNow: Array<string>,
+  now: Date,
+  opts?: { maxAgeDays?: number | undefined; max?: number | undefined },
+): DatabaseServerMemberKeys {
+  const nowTime: number =
+    now instanceof Date && Number.isFinite(now.getTime())
+      ? now.getTime()
+      : Date.now();
+
+  const maxAgeDays: number =
+    typeof opts?.maxAgeDays === "number" && opts.maxAgeDays > 0
+      ? opts.maxAgeDays
+      : DEFAULT_MEMBER_KEY_MAX_AGE_DAYS;
+  const max: number =
+    typeof opts?.max === "number" && opts.max >= 1
+      ? Math.floor(opts.max)
+      : DEFAULT_MEMBER_KEY_MAX_COUNT;
+
+  const entries: Map<string, number> = readMemberKeys(existing);
+
+  for (const [key, time] of entries) {
+    if (time > nowTime) {
+      entries.set(key, nowTime);
+    }
+  }
+
+  for (const raw of Array.isArray(seenNow) ? seenNow : []) {
+    const key: string | null = normalizeEntityKey(raw);
+    if (key !== null) {
+      entries.set(key, nowTime);
+    }
+  }
+
+  const cutoff: number = nowTime - maxAgeDays * DAY_IN_MS;
+
+  const merged: DatabaseServerMemberKeys = {};
+  let count: number = 0;
+  for (const entry of sortByRecency(entries)) {
+    if (entry.time < cutoff) {
+      continue;
+    }
+    if (count >= max) {
+      break;
+    }
+    merged[entry.key] = new Date(entry.time).toISOString();
+    count++;
+  }
+
+  return merged;
+}
+
+/*
+ * A row id as text: a string or anything with a string form (an ObjectID),
+ * trimmed; "" when there is none.
+ */
+function databaseServerIdText(value: unknown): string {
+  if (typeof value === "string") {
+    return value.trim();
+  }
+  if (value && typeof value === "object") {
+    const text: unknown = (value as { toString: () => unknown }).toString();
+    return typeof text === "string" && text !== "[object Object]"
+      ? text.trim()
+      : "";
+  }
+  return "";
+}
+
+/**
+ * Every entity key that belongs to a DatabaseServer: its row key (when the
+ * row's id is given), then the `database.server` key of each parseable
+ * stored endpoint (engine default port applied, as ingest does), then its
+ * member keys (valid 16-hex keys only, most recent first). Deduped, stable,
+ * capped at 200 with the row key first and endpoint keys next, so a large
+ * member set never evicts either.
+ *
+ * An empty result means "nothing to query" — callers must NEVER turn it into
+ * an empty `Includes` (that drops the predicate and scopes to the whole
+ * project).
+ */
+export function getDatabaseServerSignalEntityKeys(input: {
+  projectId: string;
+  /*
+   * The row's id. Its row key is what ingest stamps on telemetry linked to
+   * the row by `oneuptime.database.server.id`; leave it out only for a key
+   * set that must not include linked telemetry (the member-only runtime
+   * scope).
+   */
+  databaseServerId?: string | { toString: () => string } | null | undefined;
+  endpoints:
+    | Array<string | { endpoint?: string | null | undefined }>
+    | null
+    | undefined;
+  dbSystem?: string | null | undefined;
+  memberEntityKeys: unknown;
+}): Array<string> {
+  const projectId: string =
+    input && typeof input.projectId === "string" ? input.projectId.trim() : "";
+  if (!projectId) {
+    return [];
+  }
+
+  const keys: Array<string> = [];
+  const seen: Set<string> = new Set<string>();
+  const push: (key: string) => void = (key: string): void => {
+    if (keys.length < MAX_DATABASE_SERVER_ENTITY_KEYS && !seen.has(key)) {
+      seen.add(key);
+      keys.push(key);
+    }
+  };
+
+  const databaseServerId: string = databaseServerIdText(input.databaseServerId);
+  if (databaseServerId) {
+    push(keyForDatabaseServerRow(projectId, databaseServerId));
+  }
+
+  for (const item of Array.isArray(input.endpoints) ? input.endpoints : []) {
+    const value: unknown =
+      item && typeof item === "object"
+        ? (item as { endpoint?: unknown }).endpoint
+        : item;
+    const endpoint: DatabaseEndpoint | null = parseDatabaseEndpointString(
+      value,
+      { system: input.dbSystem || "" },
+    );
+    if (endpoint) {
+      push(keyForDatabaseEndpoint(projectId, endpoint));
+    }
+  }
+
+  for (const entry of sortByRecency(
+    readMemberKeys(input.memberEntityKeys, true),
+  )) {
+    push(entry.key);
+  }
+
+  return keys;
+}

package/Utils/Telemetry/EntityKey.ts

@@ -222,6 +222,182 @@ export function keyForVMwareVCenter(
   });
 }
 
+/*
+ * Mirrors the resolvers' `str` / `addIfPresent`: a blank or whitespace-only
+ * value is not identity-bearing, so it must be left out of the identity set
+ * rather than hashed as "".
+ */
+function isIdentityBearing(value: string | null | undefined): value is string {
+  return (
+    value !== undefined && value !== null && String(value).trim().length > 0
+  );
+}
+
+/**
+ * `k8s.cluster.name` + `k8s.namespace.name` + `k8s.pod.name` is the pod
+ * identity (mirrors the k8s.pod resolver in `InventoryItem.extractEntities`,
+ * which is name-preferring and folds in the cluster and namespace only when
+ * the resource carries them). Pass the cluster's `clusterIdentifier`, the
+ * pod's namespace and its name.
+ */
+export function keyForKubernetesPod(
+  projectId: string,
+  data: {
+    clusterName?: string | null | undefined;
+    namespace?: string | null | undefined;
+    podName: string;
+  },
+): string {
+  const identifyingAttributes: Dictionary<string> = {};
+
+  if (isIdentityBearing(data.clusterName)) {
+    identifyingAttributes["k8s.cluster.name"] = data.clusterName;
+  }
+  if (isIdentityBearing(data.namespace)) {
+    identifyingAttributes["k8s.namespace.name"] = data.namespace;
+  }
+  identifyingAttributes["k8s.pod.name"] = data.podName;
+
+  return computeEntityKey({
+    projectId,
+    entityType: EntityType.KubernetesPod,
+    identifyingAttributes,
+  });
+}
+
+/**
+ * `k8s.cluster.name` + `k8s.namespace.name` + `k8s.deployment.name` is the
+ * deployment identity (mirrors the k8s.deployment resolver). Pass the
+ * cluster's `clusterIdentifier`, the namespace and the Deployment name.
+ */
+export function keyForKubernetesDeployment(
+  projectId: string,
+  data: {
+    clusterName?: string | null | undefined;
+    namespace?: string | null | undefined;
+    deploymentName: string;
+  },
+): string {
+  const identifyingAttributes: Dictionary<string> = {};
+
+  if (isIdentityBearing(data.clusterName)) {
+    identifyingAttributes["k8s.cluster.name"] = data.clusterName;
+  }
+  if (isIdentityBearing(data.namespace)) {
+    identifyingAttributes["k8s.namespace.name"] = data.namespace;
+  }
+  identifyingAttributes["k8s.deployment.name"] = data.deploymentName;
+
+  return computeEntityKey({
+    projectId,
+    entityType: EntityType.KubernetesDeployment,
+    identifyingAttributes,
+  });
+}
+
+/**
+ * `container.id` is the container identity (mirrors the container resolver
+ * — id-only). Pass the raw `container.id` attribute value (the full id, as
+ * docker_stats / the container log path report it).
+ *
+ * Lives in this isomorphic module (it used to be server-only, in the
+ * server's entity resolver, which still re-exports it) because the Databases
+ * product computes a Docker/Podman database's member keys in the browser as
+ * well as in the worker.
+ */
+export function keyForContainer(
+  projectId: string,
+  containerId: string,
+): string {
+  return computeEntityKey({
+    projectId,
+    entityType: EntityType.Container,
+    identifyingAttributes: { "container.id": containerId },
+  });
+}
+
+/**
+ * Key for a database SERVER endpoint — the `database.server` membership key
+ * ingest appends to DB client spans, `db.client.*` datapoints and DB
+ * receiver batches that did not resolve to a row by its id, and the
+ * Databases product queries by (next to the row's own key,
+ * `keyForDatabaseServerRow`).
+ *
+ * Pass an endpoint already canonicalized by
+ * `Types/DatabaseServer/DatabaseEndpoint` (canonical host, default port
+ * applied, cluster qualifier only where it applies). This module takes
+ * primitives rather than importing that one so it stays a leaf.
+ *
+ * Deliberately ENGINE-AGNOSTIC: `db.system.name` is not part of the identity.
+ * `host:port` already names one listener, and wire-compatible engines report
+ * a different system than the server's own (CockroachDB / YugabyteDB clients
+ * say `postgresql`, OpenSearch clients say `elasticsearch`), so including it
+ * would split one server across keys.
+ */
+export function keyForDatabaseEndpoint(
+  projectId: string,
+  endpoint: {
+    host: string;
+    port: number | null;
+    kubernetesClusterName?: string | null | undefined;
+  },
+): string {
+  const identifyingAttributes: Dictionary<string> = {
+    "server.address": endpoint.host,
+  };
+
+  if (endpoint.port !== null && endpoint.port !== undefined) {
+    identifyingAttributes["server.port"] = String(endpoint.port);
+  }
+  if (isIdentityBearing(endpoint.kubernetesClusterName)) {
+    identifyingAttributes["k8s.cluster.name"] = endpoint.kubernetesClusterName;
+  }
+
+  return computeEntityKey({
+    projectId,
+    entityType: EntityType.DatabaseServer,
+    identifyingAttributes,
+  });
+}
+
+/**
+ * Identity attribute of a DatabaseServer ROW key — the attribute ingest
+ * stamps on the same telemetry (see `TelemetryUtil.getAttributesForDatabaseServerIdAndName`).
+ */
+export const DATABASE_SERVER_ROW_IDENTITY_ATTRIBUTE: string =
+  "oneuptime.database.server.id";
+
+/**
+ * Key for one DatabaseServer ROW, the `database.server` membership key that
+ * ties telemetry to the database it was attributed to rather than to an
+ * address. Ingest appends it to every row of a batch that resolved to a
+ * DatabaseServer — as the primary entity, or only through the
+ * `oneuptime.database.server.id` stamp — and the database's own key set
+ * (`getDatabaseServerSignalEntityKeys`) always contains it.
+ *
+ * Endpoint keys alone cannot carry that: a Database Agent linked by
+ * `DATABASE_SERVER_ID` may report an address the row does not own (a
+ * cluster-local name without its cluster, a private IP, an address another
+ * database owns) or no address at all (memcached, elasticsearch), and its
+ * data would then land on no database's page — or on the wrong one.
+ *
+ * Distinct from `keyForDatabaseEndpoint` (same entity type, different
+ * identifying attribute), so a row key never equals an endpoint key. Pass
+ * the row's id; it is canonicalized (lowercased) like every identity value.
+ */
+export function keyForDatabaseServerRow(
+  projectId: string,
+  databaseServerId: string,
+): string {
+  return computeEntityKey({
+    projectId,
+    entityType: EntityType.DatabaseServer,
+    identifyingAttributes: {
+      [DATABASE_SERVER_ROW_IDENTITY_ATTRIBUTE]: databaseServerId,
+    },
+  });
+}
+
 /*
  * ---- Rows without telemetry ----------------------------------------------
  *

package/Utils/ValueFormatter.ts

@@ -731,6 +731,22 @@ export default class ValueFormatter {
       return false;
     }
 
+    /*
+     * A memory fragmentation ratio (the redis receiver's
+     * `redis.memory.fragmentation_ratio`, redis_exporter's
+     * `redis_mem_fragmentation_ratio`) is resident over allocated memory —
+     * about 1.0 when healthy, 1.5 and up when fragmented, below 1 when the
+     * host swaps — not a [0, 1] share, so it is shown as the ratio (1.62,
+     * never "162.00%"). Anchored at the end like the suffix rule below, so
+     * every other `_ratio` still renders as a percent. Also what a Value or
+     * Gauge widget on it compares its thresholds against: the ratio's own
+     * scale.
+     */
+    const memoryFragmentationRatio: RegExp = /fragmentation_ratio$/i;
+    if (memoryFragmentationRatio.test(metricName.trim())) {
+      return false;
+    }
+
     const fractionMetricSuffixRegex: RegExp =
       /[._](utilization|ratio|fraction|percent|percentage)$/i;
     return fractionMetricSuffixRegex.test(metricName);

package/build/dist/Models/DatabaseModels/Alert.js

@@ -11,6 +11,7 @@ import AlertEpisode from "./AlertEpisode";
 import AlertSeverity from "./AlertSeverity";
 import AlertState from "./AlertState";
 import CephCluster from "./CephCluster";
+import DatabaseServer from "./DatabaseServer";
 import DockerHost from "./DockerHost";
 import DockerResource from "./DockerResource";
 import PodmanHost from "./PodmanHost";
@@ -81,6 +82,7 @@ let Alert = class Alert extends BaseModel {
         this.iotFleets = undefined;
         this.dockerSwarmClusters = undefined;
         this.cephClusters = undefined;
+        this.databaseServers = undefined;
         this.dockerResources = undefined;
         this.podmanResources = undefined;
         this.services = undefined;
@@ -1161,6 +1163,58 @@ __decorate([
     }),
     __metadata("design:type", Array)
 ], Alert.prototype, "cephClusters", void 0);
+__decorate([
+    ColumnAccessControl({
+        create: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.AlertAdmin,
+            Permission.AlertMember,
+            Permission.CreateAlert,
+        ],
+        read: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.Viewer,
+            Permission.AlertAdmin,
+            Permission.AlertMember,
+            Permission.AlertViewer,
+            Permission.ReadAlert,
+        ],
+        update: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.AlertAdmin,
+            Permission.AlertMember,
+            Permission.EditAlert,
+        ],
+    }),
+    TableColumn({
+        required: false,
+        type: TableColumnType.EntityArray,
+        modelType: DatabaseServer,
+        title: "Databases",
+        description: "List of databases affected by this alert.",
+    }),
+    ManyToMany(() => {
+        return DatabaseServer;
+    }, { eager: false }),
+    JoinTable({
+        name: "AlertDatabaseServer",
+        inverseJoinColumn: {
+            name: "databaseServerId",
+            referencedColumnName: "_id",
+        },
+        joinColumn: {
+            name: "alertId",
+            referencedColumnName: "_id",
+        },
+    }),
+    __metadata("design:type", Array)
+], Alert.prototype, "databaseServers", void 0);
 __decorate([
     ColumnAccessControl({
         create: [