@oneuptime/common 10.0.84 → 10.0.85

package/Server/Services/AuditLogService.ts

@@ -39,6 +39,7 @@ const NAME_CANDIDATE_FIELDS: ReadonlyArray<string> = [
 interface CachedProjectSettings {
   enableAuditLogs: boolean;
   retentionInDays: number;
+  storeSystemEventsInAuditLogs: boolean;
   planName: PlanType | undefined;
   expiresAt: number;
 }
@@ -263,7 +264,7 @@ export class AuditLogService extends AnalyticsDatabaseService<AuditLog> {
 
   private isEligible(
     settings: CachedProjectSettings | null,
-    _props: DatabaseCommonInteractionProps,
+    props: DatabaseCommonInteractionProps,
   ): boolean {
     if (!settings) {
       return false;
@@ -273,6 +274,10 @@ export class AuditLogService extends AnalyticsDatabaseService<AuditLog> {
       return false;
     }
 
+    if (!settings.storeSystemEventsInAuditLogs && this.isSystemEvent(props)) {
+      return false;
+    }
+
     if (IsEnterpriseEdition) {
       return true;
     }
@@ -288,6 +293,15 @@ export class AuditLogService extends AnalyticsDatabaseService<AuditLog> {
     return false;
   }
 
+  private isSystemEvent(props: DatabaseCommonInteractionProps): boolean {
+    /*
+     * A system event is one not initiated by a user — no userType is set and
+     * the operation is running with root privileges (e.g. background jobs,
+     * internal cleanup tasks).
+     */
+    return !props.userType && Boolean(props.isRoot);
+  }
+
   private async getProjectSettings(
     projectId: ObjectID,
   ): Promise<CachedProjectSettings | null> {
@@ -306,6 +320,7 @@ export class AuditLogService extends AnalyticsDatabaseService<AuditLog> {
         _id: true,
         enableAuditLogs: true,
         auditLogsRetentionInDays: true,
+        storeSystemEventsInAuditLogs: true,
         planName: true,
       },
       props: { isRoot: true },
@@ -318,6 +333,9 @@ export class AuditLogService extends AnalyticsDatabaseService<AuditLog> {
     const settings: CachedProjectSettings = {
       enableAuditLogs: Boolean(project.enableAuditLogs),
       retentionInDays: project.auditLogsRetentionInDays ?? 7,
+      storeSystemEventsInAuditLogs: Boolean(
+        project.storeSystemEventsInAuditLogs,
+      ),
       planName: project.planName,
       expiresAt: now + PROJECT_SETTINGS_CACHE_TTL_MS,
     };

package/Server/Services/KubernetesContainerService.ts

@@ -0,0 +1,264 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/KubernetesContainer";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ObjectID from "../../Types/ObjectID";
+import logger from "../Utils/Logger";
+
+/*
+ * ------------------------------------------------------------------
+ * KubernetesContainerService
+ *
+ * Inventory + latest-metric writes for container rows. Mirrors the
+ * shape of KubernetesResourceService but keyed by
+ * (podNamespaceKey, podName, name) since containers don't have a
+ * top-level metadata.name of their own.
+ *
+ * Callers:
+ *   - OtelLogsIngestService            -> bulkUpsert (snapshot)
+ *   - OtelMetricsIngestService         -> bulkUpdateLatestMetrics
+ *   - CleanupStaleResources worker     -> deleteStaleForCluster
+ * ------------------------------------------------------------------
+ */
+
+export interface ParsedKubernetesContainer {
+  podNamespaceKey: string;
+  podName: string;
+  name: string;
+  image: string | null;
+  state: string | null;
+  reason: string | null;
+  isReady: boolean | null;
+  restartCount: number | null;
+  memoryLimitBytes: number | null;
+  lastSeenAt: Date;
+}
+
+export interface ContainerLatestMetric {
+  podNamespaceKey: string;
+  podName: string;
+  name: string;
+  cpuPercent: number | null;
+  memoryBytes: number | null;
+  observedAt: Date;
+}
+
+const UPSERT_BATCH_SIZE: number = 500;
+const STALE_DELETE_WARN_THRESHOLD: number = 500;
+
+const UPSERT_COLUMNS: Array<string> = [
+  "projectId",
+  "kubernetesClusterId",
+  "podNamespaceKey",
+  "podName",
+  "name",
+  "image",
+  "state",
+  "reason",
+  "isReady",
+  "restartCount",
+  "memoryLimitBytes",
+  "lastSeenAt",
+  "version",
+];
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  /**
+   * Upsert a batch of parsed containers for a single (project, cluster)
+   * pair. Uses ON CONFLICT on the UNIQUE (projectId, clusterId,
+   * podNamespaceKey, podName, name) index with a dominance guard on
+   * lastSeenAt so out-of-order ingest never regresses a newer snapshot.
+   *
+   * Note: the upsert deliberately leaves latestCpuPercent /
+   * latestMemoryBytes / metricsUpdatedAt untouched — those are owned
+   * by the separate metric write path.
+   */
+  @CaptureSpan()
+  public async bulkUpsert(data: {
+    projectId: ObjectID;
+    kubernetesClusterId: ObjectID;
+    containers: Array<ParsedKubernetesContainer>;
+  }): Promise<void> {
+    if (data.containers.length === 0) {
+      return;
+    }
+
+    for (
+      let i: number = 0;
+      i < data.containers.length;
+      i += UPSERT_BATCH_SIZE
+    ) {
+      const chunk: Array<ParsedKubernetesContainer> = data.containers.slice(
+        i,
+        i + UPSERT_BATCH_SIZE,
+      );
+
+      const valueFragments: Array<string> = [];
+      const params: Array<unknown> = [];
+      let paramIndex: number = 1;
+
+      for (const c of chunk) {
+        const placeholders: Array<string> = [];
+        for (let p: number = 0; p < UPSERT_COLUMNS.length; p++) {
+          placeholders.push(`$${paramIndex++}`);
+        }
+        valueFragments.push(`(${placeholders.join(", ")})`);
+
+        params.push(
+          data.projectId.toString(),
+          data.kubernetesClusterId.toString(),
+          c.podNamespaceKey,
+          c.podName,
+          c.name,
+          c.image,
+          c.state,
+          c.reason,
+          c.isReady,
+          c.restartCount,
+          c.memoryLimitBytes !== null && c.memoryLimitBytes !== undefined
+            ? Math.trunc(c.memoryLimitBytes).toString()
+            : null,
+          c.lastSeenAt,
+          0, // version (BaseModel @VersionColumn)
+        );
+      }
+
+      const sql: string = `
+        INSERT INTO "KubernetesContainer" (
+          "projectId", "kubernetesClusterId",
+          "podNamespaceKey", "podName", "name",
+          "image", "state", "reason", "isReady", "restartCount",
+          "memoryLimitBytes", "lastSeenAt", "version"
+        )
+        VALUES ${valueFragments.join(", ")}
+        ON CONFLICT ("projectId", "kubernetesClusterId", "podNamespaceKey", "podName", "name")
+        DO UPDATE SET
+          "image" = EXCLUDED."image",
+          "state" = EXCLUDED."state",
+          "reason" = EXCLUDED."reason",
+          "isReady" = EXCLUDED."isReady",
+          "restartCount" = EXCLUDED."restartCount",
+          "memoryLimitBytes" = EXCLUDED."memoryLimitBytes",
+          "lastSeenAt" = EXCLUDED."lastSeenAt",
+          "updatedAt" = now()
+        WHERE EXCLUDED."lastSeenAt" >= "KubernetesContainer"."lastSeenAt"
+      `;
+
+      await this.getRepository().manager.query(sql, params);
+    }
+  }
+
+  /**
+   * Hard-delete all containers in a cluster whose last snapshot is
+   * older than olderThan. Returns the number of deleted rows.
+   */
+  @CaptureSpan()
+  public async deleteStaleForCluster(data: {
+    kubernetesClusterId: ObjectID;
+    olderThan: Date;
+  }): Promise<number> {
+    const result: Array<{ affected?: number }> | { affected?: number } =
+      await this.getRepository().manager.query(
+        `DELETE FROM "KubernetesContainer" WHERE "kubernetesClusterId" = $1 AND "lastSeenAt" < $2`,
+        [data.kubernetesClusterId.toString(), data.olderThan],
+      );
+
+    let affected: number = 0;
+    if (Array.isArray(result) && result.length >= 2) {
+      const second: unknown = (result as Array<unknown>)[1];
+      if (typeof second === "number") {
+        affected = second;
+      }
+    }
+
+    if (affected > STALE_DELETE_WARN_THRESHOLD) {
+      logger.warn(
+        `KubernetesContainer cleanup deleted ${affected} stale rows for cluster ${data.kubernetesClusterId.toString()} — larger than expected; investigate agent health.`,
+      );
+    }
+
+    return affected;
+  }
+
+  /**
+   * Update latestCpuPercent/latestMemoryBytes/metricsUpdatedAt for a
+   * batch of containers. Plain UPDATE — if the snapshot row doesn't
+   * exist yet, the metric write is silently skipped; the next k8s
+   * snapshot creates the row and the next metric flush catches up.
+   *
+   * The WHERE guard ensures out-of-order metric points don't regress
+   * a newer observation.
+   */
+  @CaptureSpan()
+  public async bulkUpdateLatestMetrics(data: {
+    projectId: ObjectID;
+    kubernetesClusterId: ObjectID;
+    metrics: Array<ContainerLatestMetric>;
+  }): Promise<void> {
+    if (data.metrics.length === 0) {
+      return;
+    }
+
+    /*
+     * Build a single CTE-style UPDATE per chunk: VALUES table joined to
+     * the live table on the natural key. Cheaper than firing one UPDATE
+     * per row and atomic per chunk.
+     */
+    for (let i: number = 0; i < data.metrics.length; i += UPSERT_BATCH_SIZE) {
+      const chunk: Array<ContainerLatestMetric> = data.metrics.slice(
+        i,
+        i + UPSERT_BATCH_SIZE,
+      );
+
+      const valueFragments: Array<string> = [];
+      const params: Array<unknown> = [
+        data.projectId.toString(),
+        data.kubernetesClusterId.toString(),
+      ];
+      let paramIndex: number = 3;
+
+      for (const m of chunk) {
+        valueFragments.push(
+          `($${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}::numeric, $${paramIndex++}::bigint, $${paramIndex++}::timestamptz)`,
+        );
+        params.push(
+          m.podNamespaceKey,
+          m.podName,
+          m.name,
+          m.cpuPercent !== null && m.cpuPercent !== undefined
+            ? m.cpuPercent
+            : null,
+          m.memoryBytes !== null && m.memoryBytes !== undefined
+            ? Math.trunc(m.memoryBytes).toString()
+            : null,
+          m.observedAt,
+        );
+      }
+
+      const sql: string = `
+        UPDATE "KubernetesContainer" AS k
+        SET
+          "latestCpuPercent" = COALESCE(v."cpu", k."latestCpuPercent"),
+          "latestMemoryBytes" = COALESCE(v."mem", k."latestMemoryBytes"),
+          "metricsUpdatedAt" = v."observedAt",
+          "updatedAt" = now()
+        FROM (VALUES ${valueFragments.join(", ")})
+          AS v("ns", "pod", "name", "cpu", "mem", "observedAt")
+        WHERE
+          k."projectId" = $1
+          AND k."kubernetesClusterId" = $2
+          AND k."podNamespaceKey" = v."ns"
+          AND k."podName" = v."pod"
+          AND k."name" = v."name"
+          AND (k."metricsUpdatedAt" IS NULL OR v."observedAt" >= k."metricsUpdatedAt")
+      `;
+
+      await this.getRepository().manager.query(sql, params);
+    }
+  }
+}
+
+export default new Service();

package/Server/Services/KubernetesResourceService.ts

@@ -40,6 +40,23 @@ export interface DegradedNode {
   message: string;
 }
 
+export interface ResourceLatestMetric {
+  kind: string;
+  namespaceKey: string;
+  name: string;
+  cpuPercent: number | null;
+  memoryBytes: number | null;
+  observedAt: Date;
+  /*
+   * Optional Pod controller lineage. Read from
+   * resource.k8s.deployment.name / resource.k8s.cronjob.name on the
+   * metric stream. Persisted via COALESCE so once written they stick
+   * even if a later batch lacks the attribute.
+   */
+  controllerDeploymentName?: string | null;
+  controllerCronJobName?: string | null;
+}
+
 export interface InventorySummary {
   countsByKind: Record<string, number>;
   /*
@@ -395,6 +412,83 @@ export class Service extends DatabaseService<Model> {
     }
   }
 
+  /**
+   * Update latestCpuPercent / latestMemoryBytes / metricsUpdatedAt for
+   * a batch of resources (typically Pods or Nodes). Plain UPDATE: if
+   * the snapshot row doesn't exist yet, the metric write is silently
+   * skipped — the next k8sobjects snapshot creates the row, and the
+   * next metric flush catches up.
+   *
+   * Guarded by metricsUpdatedAt so out-of-order points don't regress
+   * a newer observation.
+   */
+  @CaptureSpan()
+  public async bulkUpdateLatestMetrics(data: {
+    projectId: ObjectID;
+    kubernetesClusterId: ObjectID;
+    metrics: Array<ResourceLatestMetric>;
+  }): Promise<void> {
+    if (data.metrics.length === 0) {
+      return;
+    }
+
+    for (let i: number = 0; i < data.metrics.length; i += UPSERT_BATCH_SIZE) {
+      const chunk: Array<ResourceLatestMetric> = data.metrics.slice(
+        i,
+        i + UPSERT_BATCH_SIZE,
+      );
+
+      const valueFragments: Array<string> = [];
+      const params: Array<unknown> = [
+        data.projectId.toString(),
+        data.kubernetesClusterId.toString(),
+      ];
+      let paramIndex: number = 3;
+
+      for (const m of chunk) {
+        valueFragments.push(
+          `($${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}::numeric, $${paramIndex++}::bigint, $${paramIndex++}::timestamptz, $${paramIndex++}, $${paramIndex++})`,
+        );
+        params.push(
+          m.kind,
+          m.namespaceKey,
+          m.name,
+          m.cpuPercent !== null && m.cpuPercent !== undefined
+            ? m.cpuPercent
+            : null,
+          m.memoryBytes !== null && m.memoryBytes !== undefined
+            ? Math.trunc(m.memoryBytes).toString()
+            : null,
+          m.observedAt,
+          m.controllerDeploymentName ?? null,
+          m.controllerCronJobName ?? null,
+        );
+      }
+
+      const sql: string = `
+        UPDATE "KubernetesResource" AS k
+        SET
+          "latestCpuPercent" = COALESCE(v."cpu", k."latestCpuPercent"),
+          "latestMemoryBytes" = COALESCE(v."mem", k."latestMemoryBytes"),
+          "metricsUpdatedAt" = v."observedAt",
+          "controllerDeploymentName" = COALESCE(v."deployName", k."controllerDeploymentName"),
+          "controllerCronJobName" = COALESCE(v."cronName", k."controllerCronJobName"),
+          "updatedAt" = now()
+        FROM (VALUES ${valueFragments.join(", ")})
+          AS v("kind", "ns", "name", "cpu", "mem", "observedAt", "deployName", "cronName")
+        WHERE
+          k."projectId" = $1
+          AND k."kubernetesClusterId" = $2
+          AND k."kind" = v."kind"
+          AND k."namespaceKey" = v."ns"
+          AND k."name" = v."name"
+          AND (k."metricsUpdatedAt" IS NULL OR v."observedAt" >= k."metricsUpdatedAt")
+      `;
+
+      await this.getRepository().manager.query(sql, params);
+    }
+  }
+
   /**
    * Hard-delete all resources in a cluster whose last snapshot is
    * older than olderThan. Returns the number of deleted rows.
@@ -639,6 +733,145 @@ export class Service extends DatabaseService<Model> {
     };
   }
 
+  /**
+   * Aggregate the latest pod CPU/memory by namespace. Used by the
+   * Namespaces list view, replacing the prior ClickHouse groupBy
+   * scan. Only counts pods whose metricsUpdatedAt is within the
+   * staleness window so we don't surface stale numbers as current.
+   */
+  @CaptureSpan()
+  public async getLatestMetricsByNamespace(data: {
+    projectId: ObjectID;
+    kubernetesClusterId: ObjectID;
+    staleAfter: Date;
+  }): Promise<Map<string, { cpuPercent: number; memoryBytes: number }>> {
+    const rows: Array<{
+      namespaceKey: string;
+      cpu: string | null;
+      mem: string | null;
+    }> = await this.getRepository().manager.query(
+      `SELECT "namespaceKey",
+              SUM("latestCpuPercent")::text AS cpu,
+              SUM("latestMemoryBytes")::text AS mem
+       FROM "KubernetesResource"
+       WHERE "projectId" = $1
+         AND "kubernetesClusterId" = $2
+         AND "kind" = 'Pod'
+         AND "deletedAt" IS NULL
+         AND "metricsUpdatedAt" IS NOT NULL
+         AND "metricsUpdatedAt" >= $3
+       GROUP BY "namespaceKey"`,
+      [
+        data.projectId.toString(),
+        data.kubernetesClusterId.toString(),
+        data.staleAfter,
+      ],
+    );
+
+    const out: Map<string, { cpuPercent: number; memoryBytes: number }> =
+      new Map();
+    for (const row of rows) {
+      out.set(row.namespaceKey || "", {
+        cpuPercent: row.cpu ? parseFloat(row.cpu) || 0 : 0,
+        memoryBytes: row.mem ? parseInt(row.mem, 10) || 0 : 0,
+      });
+    }
+    return out;
+  }
+
+  /**
+   * Aggregate the latest pod CPU/memory by owner (Deployment /
+   * StatefulSet / DaemonSet / Job / CronJob).
+   *
+   * Direct-owner kinds (StatefulSet, DaemonSet, Job) read from the
+   * Pod's ownerReferences JSONB. Indirect-owner kinds (Deployment,
+   * CronJob) read from the denormalized controllerDeploymentName /
+   * controllerCronJobName columns populated by the metric ingest
+   * path — Pods don't directly own to those kinds, so we can't walk
+   * ownerReferences for them.
+   *
+   * Returns a Map keyed by owner name. Pods without recent metrics
+   * (metricsUpdatedAt past the staleness cutoff) are excluded so the
+   * sum reflects "right now," not "ever observed."
+   */
+  @CaptureSpan()
+  public async getLatestMetricsByOwner(data: {
+    projectId: ObjectID;
+    kubernetesClusterId: ObjectID;
+    ownerKind: string;
+    staleAfter: Date;
+  }): Promise<Map<string, { cpuPercent: number; memoryBytes: number }>> {
+    let rows: Array<{
+      ownerName: string;
+      cpu: string | null;
+      mem: string | null;
+    }>;
+
+    if (data.ownerKind === "Deployment" || data.ownerKind === "CronJob") {
+      const column: string =
+        data.ownerKind === "Deployment"
+          ? "controllerDeploymentName"
+          : "controllerCronJobName";
+      rows = await this.getRepository().manager.query(
+        `SELECT
+           "${column}" AS "ownerName",
+           SUM("latestCpuPercent")::text AS cpu,
+           SUM("latestMemoryBytes")::text AS mem
+         FROM "KubernetesResource"
+         WHERE "projectId" = $1
+           AND "kubernetesClusterId" = $2
+           AND "kind" = 'Pod'
+           AND "deletedAt" IS NULL
+           AND "metricsUpdatedAt" IS NOT NULL
+           AND "metricsUpdatedAt" >= $3
+           AND "${column}" IS NOT NULL
+         GROUP BY "${column}"`,
+        [
+          data.projectId.toString(),
+          data.kubernetesClusterId.toString(),
+          data.staleAfter,
+        ],
+      );
+    } else {
+      rows = await this.getRepository().manager.query(
+        `SELECT
+           (owner->>'name') AS "ownerName",
+           SUM("latestCpuPercent")::text AS cpu,
+           SUM("latestMemoryBytes")::text AS mem
+         FROM "KubernetesResource",
+              jsonb_array_elements("ownerReferences"->'items') AS owner
+         WHERE "projectId" = $1
+           AND "kubernetesClusterId" = $2
+           AND "kind" = 'Pod'
+           AND "deletedAt" IS NULL
+           AND "metricsUpdatedAt" IS NOT NULL
+           AND "metricsUpdatedAt" >= $3
+           AND "ownerReferences" IS NOT NULL
+           AND owner->>'kind' = $4
+         GROUP BY (owner->>'name')`,
+        [
+          data.projectId.toString(),
+          data.kubernetesClusterId.toString(),
+          data.staleAfter,
+          data.ownerKind,
+        ],
+      );
+    }
+
+    const out: Map<string, { cpuPercent: number; memoryBytes: number }> =
+      new Map();
+    for (const row of rows) {
+      if (!row.ownerName) {
+        continue;
+      }
+      out.set(row.ownerName, {
+        cpuPercent: row.cpu ? parseFloat(row.cpu) || 0 : 0,
+        memoryBytes: row.mem ? parseInt(row.mem, 10) || 0 : 0,
+      });
+    }
+    return out;
+  }
+
   /**
    * Helper for the cleanup worker: snapshot-interval aware cutoff.
    * 3× the 5-minute snapshot interval. Tune via CLEANUP_THRESHOLD_MINUTES.

package/Server/Services/StatusPageSubscriberService.ts

@@ -463,14 +463,14 @@ export class Service extends DatabaseService<Model> {
       logger.debug("Subscriber has an email.", {
         projectId: createdItem.projectId?.toString(),
       } as LogAttributes);
-      const isSubcriptionConfirmed: boolean = Boolean(
+      const isSubscriptionConfirmed: boolean = Boolean(
         createdItem.isSubscriptionConfirmed,
       );
-      logger.debug(`Is Subscription Confirmed: ${isSubcriptionConfirmed}`, {
+      logger.debug(`Is Subscription Confirmed: ${isSubscriptionConfirmed}`, {
         projectId: createdItem.projectId?.toString(),
       } as LogAttributes);
 
-      if (!isSubcriptionConfirmed) {
+      if (!isSubscriptionConfirmed) {
         logger.debug(
           "Subscription is not confirmed. Sending confirmation email.",
           { projectId: createdItem.projectId?.toString() } as LogAttributes,
@@ -480,7 +480,7 @@ export class Service extends DatabaseService<Model> {
         });
       }
 
-      if (isSubcriptionConfirmed && createdItem.sendYouHaveSubscribedMessage) {
+      if (isSubscriptionConfirmed && createdItem.sendYouHaveSubscribedMessage) {
         logger.debug(
           "Subscription is confirmed and sendYouHaveSubscribedMessage is true. Sending 'You have subscribed' email.",
           { projectId: createdItem.projectId?.toString() } as LogAttributes,

package/Server/Types/Database/Permissions/AccessControlPermission.ts

@@ -149,10 +149,10 @@ export default class AccessControlPermission {
       model.getAccessControlColumn();
 
     if (modelAccessControlColumnName) {
-      const accessControlIdsWhcihUserHasAccessTo: Array<ObjectID> =
+      const accessControlIdsWhichUserHasAccessTo: Array<ObjectID> =
         this.getAccessControlIdsForModel(modelType, props, type);
 
-      if (accessControlIdsWhcihUserHasAccessTo.length === 0) {
+      if (accessControlIdsWhichUserHasAccessTo.length === 0) {
         return; // The user has access to all resources, if no labels are specified.
       }
 
@@ -164,7 +164,7 @@ export default class AccessControlPermission {
       }
 
       const accessControlIdsWhichUserHasAccessToAsStrings: Array<string> =
-        accessControlIdsWhcihUserHasAccessTo.map((id: ObjectID) => {
+        accessControlIdsWhichUserHasAccessTo.map((id: ObjectID) => {
           return id.toString();
         }) || [];