@oneuptime/common 10.0.67 → 10.0.69

package/Server/Services/DatabaseService.ts

@@ -772,6 +772,31 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
         );
       }
 
+      if (
+        !createBy.props.ignoreHooks &&
+        this.getModel().enableAuditLogOn?.create
+      ) {
+        /*
+         * Lazy require to avoid circular dependency between DatabaseService and
+         * AuditLogService (which depends on ProjectService/UserService, both of
+         * which extend DatabaseService). A top-level import leaves
+         * DatabaseService undefined at class-extension time for subclasses.
+         */
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const auditLogService: {
+          recordCreate: (data: {
+            model: TBaseModel;
+            createdItem: TBaseModel;
+            props: DatabaseCommonInteractionProps;
+          }) => Promise<void>;
+        } = require("./AuditLogService").default;
+        await auditLogService.recordCreate({
+          model: this.getModel(),
+          createdItem: createBy.data,
+          props: createBy.props,
+        });
+      }
+
       return createBy.data;
     } catch (error) {
       await this.onCreateError(error as Exception);
@@ -1117,6 +1142,18 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
         (select as any)[this.getModel().getTenantColumn() as string] = true;
       }
 
+      /*
+       * If audit logging on delete is enabled, fetch all scalar columns so we
+       * can record a full snapshot of the record before it is deleted.
+       */
+      if (this.getModel().enableAuditLogOn?.delete) {
+        const allColumns: Array<string> =
+          this.getModel().getTableColumns().columns;
+        for (const columnName of allColumns) {
+          (select as any)[columnName] = true;
+        }
+      }
+
       const items: Array<TBaseModel> = await this._findBy({
         query: beforeDeleteBy.query,
         skip: beforeDeleteBy.skip.toNumber(),
@@ -1199,6 +1236,28 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
         );
       }
 
+      if (this.getModel().enableAuditLogOn?.delete && items.length > 0) {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const auditLogService: {
+          recordDelete: (args: {
+            model: TBaseModel;
+            deletedItem: TBaseModel;
+            itemId: ObjectID;
+            props: DatabaseCommonInteractionProps;
+          }) => Promise<void>;
+        } = require("./AuditLogService").default;
+        for (const item of items) {
+          if (item.id) {
+            await auditLogService.recordDelete({
+              model: this.getModel(),
+              deletedItem: item,
+              itemId: item.id,
+              props: deleteBy.props,
+            });
+          }
+        }
+      }
+
       return numberOfDocsAffected;
     } catch (error) {
       await this.onDeleteError(error as Exception);
@@ -1533,6 +1592,26 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
           true;
       }
 
+      /*
+       * When audit logging on update is enabled, ensure the resource's display
+       * name is loaded on the `before` snapshot so the audit entry records the
+       * human-readable resource name even when the update doesn't touch it.
+       */
+      if (this.getModel().enableAuditLogOn?.update) {
+        const nameCandidates: ReadonlyArray<string> = [
+          "name",
+          "title",
+          "displayName",
+        ];
+        const modelColumns: Array<string> =
+          this.getModel().getTableColumns().columns;
+        for (const candidate of nameCandidates) {
+          if (modelColumns.includes(candidate)) {
+            (selectColumns as any)[candidate] = true;
+          }
+        }
+      }
+
       const items: Array<TBaseModel> = await this._findBy({
         query: beforeUpdateBy.query,
         skip: updateBy.skip.toNumber(),
@@ -1582,6 +1661,30 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
             );
           }
         }
+
+        if (
+          this.getModel().enableAuditLogOn?.update &&
+          !this.hasSameValues({ item, updatedItem }) &&
+          item.id
+        ) {
+          // eslint-disable-next-line @typescript-eslint/no-var-requires
+          const auditLogService: {
+            recordUpdate: (args: {
+              model: TBaseModel;
+              before: TBaseModel;
+              updatedFields: JSONObject;
+              itemId: ObjectID;
+              props: DatabaseCommonInteractionProps;
+            }) => Promise<void>;
+          } = require("./AuditLogService").default;
+          await auditLogService.recordUpdate({
+            model: this.getModel(),
+            before: item,
+            updatedFields: data as JSONObject,
+            itemId: item.id,
+            props: updateBy.props,
+          });
+        }
       }
 
       /*

package/Server/Services/Index.ts

@@ -36,6 +36,7 @@ import LabelService from "./LabelService";
 import KubernetesClusterService from "./KubernetesClusterService";
 import DockerHostService from "./DockerHostService";
 import LlmProviderService from "./LlmProviderService";
+import AuditLogService from "./AuditLogService";
 import LogService from "./LogService";
 import MailService from "./MailService";
 import MetricService from "./MetricService";
@@ -447,6 +448,7 @@ export const AnalyticsServices: Array<
   MonitorLogService,
   ProfileService,
   ProfileSampleService,
+  AuditLogService,
 ];
 
 export default services;

package/Server/Services/KubernetesResourceService.ts

@@ -22,10 +22,28 @@ import logger from "../Utils/Logger";
 
 export type { ParsedKubernetesResource };
 
+export interface DegradedPod {
+  name: string;
+  namespace: string;
+  phase: string;
+  reason: string;
+  message: string;
+}
+
+export interface DegradedNode {
+  name: string;
+  isReady: boolean;
+  hasMemoryPressure: boolean;
+  hasDiskPressure: boolean;
+  hasPidPressure: boolean;
+  reason: string;
+  message: string;
+}
+
 export interface InventorySummary {
   countsByKind: Record<string, number>;
   /*
-   * Sum of `jsonb_array_length(spec->'containers')` across all pods in
+   * Sum of the denormalized containerCount column across all pods in
    * the cluster. Containers aren't a top-level kind in the inventory,
    * so we derive the total server-side so the sidebar badge and the
    * Containers page agree.
@@ -47,6 +65,201 @@ export interface InventorySummary {
     diskPressure: number;
     pidPressure: number;
   };
+  /*
+   * Top offenders that explain a Degraded/Unhealthy cluster state. Capped
+   * so a pathological cluster can't blow up the overview payload; the
+   * dedicated Pods/Nodes pages are the source of truth for the full list.
+   */
+  degradedPods: Array<DegradedPod>;
+  degradedNodes: Array<DegradedNode>;
+}
+
+const DEGRADED_SAMPLE_LIMIT: number = 20;
+
+/*
+ * Pull the first meaningful reason/message off a pod's status block.
+ * KubernetesInventoryExtractor stores containerStatuses as an array of
+ * { name, ready, state: "running"|"waiting"|"terminated", reason, message, ... }.
+ * A waiting container with a reason (ImagePullBackOff, CrashLoopBackOff,
+ * CreateContainerConfigError, ...) is exactly what the user needs to see,
+ * so we surface that first. We fall back to terminated reasons (OOMKilled,
+ * Error, ContainerCannotRun) and then to status-level conditions.
+ */
+function buildDegradedPod(row: {
+  name: string;
+  namespaceKey: string;
+  phase: string | null;
+  status: unknown;
+}): DegradedPod {
+  const status: Record<string, unknown> =
+    row.status && typeof row.status === "object"
+      ? (row.status as Record<string, unknown>)
+      : {};
+
+  let reason: string = "";
+  let message: string = "";
+
+  const containerStatuses: Array<Record<string, unknown>> = Array.isArray(
+    status["containerStatuses"],
+  )
+    ? (status["containerStatuses"] as Array<Record<string, unknown>>)
+    : [];
+  const initContainerStatuses: Array<Record<string, unknown>> = Array.isArray(
+    status["initContainerStatuses"],
+  )
+    ? (status["initContainerStatuses"] as Array<Record<string, unknown>>)
+    : [];
+
+  const scanForReason: (
+    list: Array<Record<string, unknown>>,
+    targetState: string,
+  ) => { reason: string; message: string } | null = (list, targetState) => {
+    for (const cs of list) {
+      if (cs["state"] !== targetState) {
+        continue;
+      }
+      const r: unknown = cs["reason"];
+      if (typeof r === "string" && r) {
+        const m: unknown = cs["message"];
+        return {
+          reason: r,
+          message: typeof m === "string" ? m : "",
+        };
+      }
+    }
+    return null;
+  };
+
+  const waitingHit: { reason: string; message: string } | null =
+    scanForReason(containerStatuses, "waiting") ||
+    scanForReason(initContainerStatuses, "waiting");
+  const terminatedHit: { reason: string; message: string } | null = waitingHit
+    ? null
+    : scanForReason(containerStatuses, "terminated") ||
+      scanForReason(initContainerStatuses, "terminated");
+  const hit: { reason: string; message: string } | null =
+    waitingHit || terminatedHit;
+
+  if (hit) {
+    reason = hit.reason;
+    message = hit.message;
+  } else {
+    // Fall back to the pod-level reason/message fields set by the scheduler
+    // (e.g. "Unschedulable" with "0/3 nodes are available: ...").
+    const topReason: unknown = status["reason"];
+    const topMessage: unknown = status["message"];
+    if (typeof topReason === "string") {
+      reason = topReason;
+    }
+    if (typeof topMessage === "string") {
+      message = topMessage;
+    }
+
+    // If still nothing, pull from the first non-True condition.
+    if (!reason) {
+      const conditions: Array<Record<string, unknown>> = Array.isArray(
+        status["conditions"],
+      )
+        ? (status["conditions"] as Array<Record<string, unknown>>)
+        : [];
+      for (const cond of conditions) {
+        if (cond["status"] !== "True") {
+          const r: unknown = cond["reason"];
+          const m: unknown = cond["message"];
+          if (typeof r === "string" && r) {
+            reason = r;
+            message = typeof m === "string" ? m : "";
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  return {
+    name: row.name,
+    namespace: row.namespaceKey || "",
+    phase: row.phase || "Unknown",
+    reason,
+    message,
+  };
+}
+
+/*
+ * For a Node: if isReady is false, the "Ready" condition carries the real
+ * story (e.g. "KubeletNotReady: PLEG is not healthy"). If only pressure
+ * flags are tripped, pick the tripped condition's reason/message.
+ */
+function buildDegradedNode(row: {
+  name: string;
+  isReady: boolean | null;
+  hasMemoryPressure: boolean | null;
+  hasDiskPressure: boolean | null;
+  hasPidPressure: boolean | null;
+  status: unknown;
+}): DegradedNode {
+  const status: Record<string, unknown> =
+    row.status && typeof row.status === "object"
+      ? (row.status as Record<string, unknown>)
+      : {};
+
+  const conditions: Array<Record<string, unknown>> = Array.isArray(
+    status["conditions"],
+  )
+    ? (status["conditions"] as Array<Record<string, unknown>>)
+    : [];
+
+  const findCondition: (
+    predicate: (c: Record<string, unknown>) => boolean,
+  ) => Record<string, unknown> | null = (predicate) => {
+    for (const c of conditions) {
+      if (predicate(c)) {
+        return c;
+      }
+    }
+    return null;
+  };
+
+  let picked: Record<string, unknown> | null = null;
+  if (row.isReady === false) {
+    picked = findCondition((c: Record<string, unknown>) => {
+      return c["type"] === "Ready" && c["status"] !== "True";
+    });
+  }
+  if (!picked && row.hasMemoryPressure === true) {
+    picked = findCondition((c: Record<string, unknown>) => {
+      return c["type"] === "MemoryPressure" && c["status"] === "True";
+    });
+  }
+  if (!picked && row.hasDiskPressure === true) {
+    picked = findCondition((c: Record<string, unknown>) => {
+      return c["type"] === "DiskPressure" && c["status"] === "True";
+    });
+  }
+  if (!picked && row.hasPidPressure === true) {
+    picked = findCondition((c: Record<string, unknown>) => {
+      return c["type"] === "PIDPressure" && c["status"] === "True";
+    });
+  }
+
+  const reason: string =
+    picked && typeof picked["reason"] === "string"
+      ? (picked["reason"] as string)
+      : "";
+  const message: string =
+    picked && typeof picked["message"] === "string"
+      ? (picked["message"] as string)
+      : "";
+
+  return {
+    name: row.name,
+    isReady: row.isReady === true,
+    hasMemoryPressure: row.hasMemoryPressure === true,
+    hasDiskPressure: row.hasDiskPressure === true,
+    hasPidPressure: row.hasPidPressure === true,
+    reason,
+    message,
+  };
 }
 
 const UPSERT_BATCH_SIZE: number = 500;
@@ -72,6 +285,7 @@ const UPSERT_COLUMNS: Array<keyof ParsedKubernetesResource | string> = [
   "annotations",
   "ownerReferences",
   "spec",
+  "containerCount",
   "status",
   "lastSeenAt",
   "resourceCreationTimestamp",
@@ -133,6 +347,7 @@ export class Service extends DatabaseService<Model> {
           r.annotations ? JSON.stringify(r.annotations) : null,
           r.ownerReferences ? JSON.stringify(r.ownerReferences) : null,
           r.spec ? JSON.stringify(r.spec) : null,
+          r.containerCount,
           r.status ? JSON.stringify(r.status) : null,
           r.lastSeenAt,
           r.resourceCreationTimestamp,
@@ -145,7 +360,7 @@ export class Service extends DatabaseService<Model> {
           "projectId", "kubernetesClusterId", "kind", "namespaceKey", "name",
           "uid", "phase", "isReady",
           "hasMemoryPressure", "hasDiskPressure", "hasPidPressure",
-          "labels", "annotations", "ownerReferences", "spec", "status",
+          "labels", "annotations", "ownerReferences", "spec", "containerCount", "status",
           "lastSeenAt", "resourceCreationTimestamp", "version"
         )
         VALUES ${valueFragments.join(", ")}
@@ -161,6 +376,7 @@ export class Service extends DatabaseService<Model> {
           "annotations" = EXCLUDED."annotations",
           "ownerReferences" = EXCLUDED."ownerReferences",
           "spec" = EXCLUDED."spec",
+          "containerCount" = EXCLUDED."containerCount",
           "status" = EXCLUDED."status",
           "lastSeenAt" = EXCLUDED."lastSeenAt",
           "resourceCreationTimestamp" = EXCLUDED."resourceCreationTimestamp",
@@ -218,7 +434,14 @@ export class Service extends DatabaseService<Model> {
     const manager: ReturnType<Service["getRepository"]>["manager"] =
       this.getRepository().manager;
 
-    const [kindRows, podRows, nodeRows, containerRows]: [
+    const [
+      kindRows,
+      podRows,
+      nodeRows,
+      containerRows,
+      degradedPodRows,
+      degradedNodeRows,
+    ]: [
       Array<{ kind: string; count: string }>,
       Array<{ phase: string | null; count: string }>,
       Array<{
@@ -229,6 +452,20 @@ export class Service extends DatabaseService<Model> {
         pidPressure: string;
       }>,
       Array<{ total: string }>,
+      Array<{
+        name: string;
+        namespaceKey: string;
+        phase: string | null;
+        status: unknown;
+      }>,
+      Array<{
+        name: string;
+        isReady: boolean | null;
+        hasMemoryPressure: boolean | null;
+        hasDiskPressure: boolean | null;
+        hasPidPressure: boolean | null;
+        status: unknown;
+      }>,
     ] = await Promise.all([
       manager.query(
         `SELECT "kind", COUNT(*)::text AS count
@@ -256,15 +493,61 @@ export class Service extends DatabaseService<Model> {
         [data.projectId.toString(), data.kubernetesClusterId.toString()],
       ),
       manager.query(
-        `SELECT COALESCE(SUM(
-           CASE WHEN jsonb_typeof("spec"->'containers') = 'array'
-                THEN jsonb_array_length("spec"->'containers')
-                ELSE 0 END
-         ), 0)::text AS total
+        /*
+         * containerCount is cached on the row during ingest
+         * (KubernetesInventoryExtractor sets it from
+         * spec.containers.length), so this is a plain int sum instead
+         * of a JSONB scan. Rows written before that ingest change may
+         * have NULL; SUM treats those as 0, which matches the old
+         * behavior.
+         */
+        `SELECT COALESCE(SUM("containerCount"), 0)::text AS total
          FROM "KubernetesResource"
          WHERE "projectId" = $1 AND "kubernetesClusterId" = $2 AND "kind" = 'Pod' AND "deletedAt" IS NULL`,
         [data.projectId.toString(), data.kubernetesClusterId.toString()],
       ),
+      /*
+       * Top-N offenders powering the "Why is this cluster degraded?" card.
+       * Failed first (hardest outage), then Pending, then Unknown, so the
+       * user sees the worst stuff first without having to sort client-side.
+       */
+      manager.query(
+        `SELECT "name", "namespaceKey", "phase", "status"
+         FROM "KubernetesResource"
+         WHERE "projectId" = $1
+           AND "kubernetesClusterId" = $2
+           AND "kind" = 'Pod'
+           AND "deletedAt" IS NULL
+           AND ("phase" IS NULL OR "phase" NOT IN ('Running', 'Succeeded'))
+         ORDER BY
+           CASE "phase"
+             WHEN 'Failed' THEN 0
+             WHEN 'Pending' THEN 1
+             ELSE 2
+           END,
+           "lastSeenAt" DESC
+         LIMIT ${DEGRADED_SAMPLE_LIMIT}`,
+        [data.projectId.toString(), data.kubernetesClusterId.toString()],
+      ),
+      manager.query(
+        `SELECT "name", "isReady", "hasMemoryPressure", "hasDiskPressure", "hasPidPressure", "status"
+         FROM "KubernetesResource"
+         WHERE "projectId" = $1
+           AND "kubernetesClusterId" = $2
+           AND "kind" = 'Node'
+           AND "deletedAt" IS NULL
+           AND (
+             "isReady" IS FALSE
+             OR "hasMemoryPressure" IS TRUE
+             OR "hasDiskPressure" IS TRUE
+             OR "hasPidPressure" IS TRUE
+           )
+         ORDER BY
+           CASE WHEN "isReady" IS FALSE THEN 0 ELSE 1 END,
+           "lastSeenAt" DESC
+         LIMIT ${DEGRADED_SAMPLE_LIMIT}`,
+        [data.projectId.toString(), data.kubernetesClusterId.toString()],
+      ),
     ]);
 
     const countsByKind: Record<string, number> = {};
@@ -308,6 +591,13 @@ export class Service extends DatabaseService<Model> {
     const containerCount: number =
       parseInt(containerRows[0]?.total || "0", 10) || 0;
 
+    const degradedPods: Array<DegradedPod> = degradedPodRows.map((row) => {
+      return buildDegradedPod(row);
+    });
+    const degradedNodes: Array<DegradedNode> = degradedNodeRows.map((row) => {
+      return buildDegradedNode(row);
+    });
+
     return {
       countsByKind,
       containerCount,
@@ -321,6 +611,8 @@ export class Service extends DatabaseService<Model> {
         diskPressure: parseInt(nodeRow?.diskPressure || "0", 10) || 0,
         pidPressure: parseInt(nodeRow?.pidPressure || "0", 10) || 0,
       },
+      degradedPods,
+      degradedNodes,
     };
   }
 

package/Server/Utils/VM/VMRunner.ts

@@ -224,7 +224,7 @@ function createSandboxProxy(
  * Recursively unwraps sandbox proxies in a return value so the host code
  * receives original objects (e.g. Buffers that pass `instanceof` checks).
  */
-function deepUnwrapProxies(
+export function deepUnwrapProxies(
   value: unknown,
   visited?: WeakSet<GenericObject>,
 ): unknown {
@@ -469,33 +469,50 @@ export default class VMRunner {
       })()`;
 
     try {
-      /*
-       * vm timeout only covers synchronous CPU time, so wrap with
-       * Promise.race to also cover async operations (network, timers, etc.)
-       */
-      const vmPromise: Promise<unknown> = vm.runInContext(script, sandbox, {
-        timeout: timeout,
-      });
-
-      const overallTimeout: Promise<never> = new Promise(
-        (_resolve: (value: never) => void, reject: (reason: Error) => void) => {
-          const handle: NodeJS.Timeout = global.setTimeout(() => {
-            reject(new Error("Script execution timed out"));
-          }, timeout + 5000);
-          // Don't let this timer keep the process alive
-          handle.unref();
-        },
-      );
+      let returnVal: unknown;
+      let scriptError: Error | undefined;
+
+      try {
+        /*
+         * vm timeout only covers synchronous CPU time, so wrap with
+         * Promise.race to also cover async operations (network, timers, etc.)
+         */
+        const vmPromise: Promise<unknown> = vm.runInContext(script, sandbox, {
+          timeout: timeout,
+        });
 
-      const returnVal: unknown = await Promise.race([
-        vmPromise,
-        overallTimeout,
-      ]);
+        const overallTimeout: Promise<never> = new Promise(
+          (
+            _resolve: (value: never) => void,
+            reject: (reason: Error) => void,
+          ) => {
+            const handle: NodeJS.Timeout = global.setTimeout(() => {
+              reject(new Error("Script execution timed out"));
+            }, timeout + 5000);
+            // Don't let this timer keep the process alive
+            handle.unref();
+          },
+        );
+
+        returnVal = await Promise.race([vmPromise, overallTimeout]);
+      } catch (err: unknown) {
+        /*
+         * Capture user-thrown errors (including timeouts) so the caller can
+         * still access side-channel data collected before the throw — e.g.
+         * screenshots assigned to a host-realm object passed via `context`.
+         * Rethrowing here would discard those partial results.
+         */
+        scriptError =
+          err instanceof Error
+            ? err
+            : new Error(typeof err === "string" ? err : String(err));
+      }
 
       return {
         returnValue: deepUnwrapProxies(returnVal),
         logMessages,
         capturedMetrics,
+        scriptError,
       };
     } finally {
       // Clean up any lingering timers to prevent resource leaks

package/Types/AnalyticsDatabase/AnalyticsTableName.ts

@@ -6,6 +6,7 @@ enum AnalyticsTableName {
   MonitorLog = "MonitorLogV2",
   Profile = "ProfileItemV2",
   ProfileSample = "ProfileSampleItemV2",
+  AuditLog = "AuditLogV1",
 }
 
 export default AnalyticsTableName;

package/Types/AuditLog/AuditLogAction.ts

@@ -0,0 +1,7 @@
+enum AuditLogAction {
+  Create = "Create",
+  Update = "Update",
+  Delete = "Delete",
+}
+
+export default AuditLogAction;

package/Types/BaseDatabase/EnableAuditLogOn.ts

@@ -0,0 +1,5 @@
+export default interface EnableAuditLogOn {
+  create?: boolean | undefined;
+  update?: boolean | undefined;
+  delete?: boolean | undefined;
+}

package/Types/Database/EnableAuditLog.ts

@@ -0,0 +1,18 @@
+import EnableAuditLogOn from "../BaseDatabase/EnableAuditLogOn";
+import GenericFunction from "../GenericFunction";
+
+export default (
+  enableAuditLogOn: EnableAuditLogOn = {
+    create: true,
+    update: true,
+    delete: true,
+  },
+) => {
+  return (ctr: GenericFunction) => {
+    ctr.prototype.enableAuditLogOn = {
+      create: enableAuditLogOn.create ?? true,
+      update: enableAuditLogOn.update ?? true,
+      delete: enableAuditLogOn.delete ?? true,
+    };
+  };
+};

package/Types/IsolatedVM/ReturnResult.ts

@@ -4,4 +4,10 @@ export default interface ReturnResult {
   returnValue: any;
   logMessages: string[];
   capturedMetrics: CapturedMetric[];
+  /**
+   * Populated when user-supplied code threw (or timed out). The runner still
+   * returns collected side-channel data (logs, metrics, and any host-realm
+   * context objects the caller passed in) so partial state survives the throw.
+   */
+  scriptError?: Error | undefined;
 }