@oneuptime/common 10.0.43 → 10.0.45

package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts

@@ -1,4 +1,5 @@
 import logger from "../Logger";
+import LogAggregationService from "../../Services/LogAggregationService";
 import VMUtil from "../VM/VMAPI";
 import APIRequestCriteria from "./Criteria/APIRequestCriteria";
 import CustomCodeMonitoringCriteria from "./Criteria/CustomCodeMonitorCriteria";
@@ -116,7 +117,7 @@ export default class MonitorCriteriaEvaluator {
 `;
 
         const contextBlock: string | null =
-          MonitorCriteriaEvaluator.buildRootCauseContext({
+          await MonitorCriteriaEvaluator.buildRootCauseContext({
             dataToProcess: input.dataToProcess,
             monitorStep: input.monitorStep,
             monitor: input.monitor,
@@ -557,14 +558,16 @@ ${contextBlock}
     return null;
   }
 
-  private static buildRootCauseContext(input: {
+  private static async buildRootCauseContext(input: {
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;
     monitor: Monitor;
-  }): string | null {
+  }): Promise<string | null> {
     // Handle Kubernetes monitors with rich resource context
     if (input.monitor.monitorType === MonitorType.Kubernetes) {
-      return MonitorCriteriaEvaluator.buildKubernetesRootCauseContext(input);
+      return await MonitorCriteriaEvaluator.buildKubernetesRootCauseContext(
+        input,
+      );
     }
 
     const requestDetails: Array<string> = [];
@@ -675,11 +678,11 @@ ${contextBlock}
     return sections.join("\n");
   }
 
-  private static buildKubernetesRootCauseContext(input: {
+  private static async buildKubernetesRootCauseContext(input: {
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;
     monitor: Monitor;
-  }): string | null {
+  }): Promise<string | null> {
     const metricResponse: MetricMonitorResponse =
       input.dataToProcess as MetricMonitorResponse;
 
@@ -713,51 +716,121 @@ ${contextBlock}
     if (breakdown.affectedResources && breakdown.affectedResources.length > 0) {
       const resourceLines: Array<string> = [];
 
-      // Sort by metric value descending (worst first)
+      // Sort by metric value descending (worst first) and filter out zero-value resources
       const sortedResources: Array<KubernetesAffectedResource> = [
         ...breakdown.affectedResources,
-      ].sort((a: KubernetesAffectedResource, b: KubernetesAffectedResource) => {
-        return b.metricValue - a.metricValue;
-      });
+      ]
+        .filter((r: KubernetesAffectedResource) => {
+          return r.metricValue > 0;
+        })
+        .sort(
+          (a: KubernetesAffectedResource, b: KubernetesAffectedResource) => {
+            return b.metricValue - a.metricValue;
+          },
+        );
+
+      if (sortedResources.length === 0) {
+        return sections.join("\n");
+      }
 
       // Show top 10 affected resources
       const resourcesToShow: Array<KubernetesAffectedResource> =
         sortedResources.slice(0, 10);
 
+      // Determine which columns are present across all resources
+      const hasNamespace: boolean = resourcesToShow.some(
+        (r: KubernetesAffectedResource) => {
+          return r.namespace;
+        },
+      );
+      const hasWorkload: boolean = resourcesToShow.some(
+        (r: KubernetesAffectedResource) => {
+          return r.workloadType && r.workloadName;
+        },
+      );
+      const hasPod: boolean = resourcesToShow.some(
+        (r: KubernetesAffectedResource) => {
+          return r.podName;
+        },
+      );
+      const hasContainer: boolean = resourcesToShow.some(
+        (r: KubernetesAffectedResource) => {
+          return r.containerName;
+        },
+      );
+      const hasNode: boolean = resourcesToShow.some(
+        (r: KubernetesAffectedResource) => {
+          return r.nodeName;
+        },
+      );
+
+      // Build table header
+      const headerCells: Array<string> = [];
+      if (hasNamespace) {
+        headerCells.push("Namespace");
+      }
+      if (hasWorkload) {
+        headerCells.push("Workload Type");
+        headerCells.push("Workload");
+      }
+      if (hasPod) {
+        headerCells.push("Pod");
+      }
+      if (hasContainer) {
+        headerCells.push("Container");
+      }
+      if (hasNode) {
+        headerCells.push("Node");
+      }
+      headerCells.push("Value");
+
+      const headerRow: string = `| ${headerCells.join(" | ")} |`;
+      const separatorRow: string = `| ${headerCells
+        .map(() => {
+          return "---";
+        })
+        .join(" | ")} |`;
+
+      resourceLines.push(headerRow);
+      resourceLines.push(separatorRow);
+
       for (const resource of resourcesToShow) {
-        const details: Array<string> = [];
+        const cells: Array<string> = [];
 
-        if (resource.namespace) {
-          details.push(`Namespace: \`${resource.namespace}\``);
+        if (hasNamespace) {
+          cells.push(resource.namespace ? `\`${resource.namespace}\`` : "-");
         }
-        if (resource.workloadType && resource.workloadName) {
-          details.push(
-            `${resource.workloadType}: \`${resource.workloadName}\``,
+        if (hasWorkload) {
+          cells.push(resource.workloadType ? `${resource.workloadType}` : "-");
+          cells.push(
+            resource.workloadName ? `\`${resource.workloadName}\`` : "-",
           );
         }
-        if (resource.podName) {
-          details.push(`Pod: \`${resource.podName}\``);
+        if (hasPod) {
+          cells.push(resource.podName ? `\`${resource.podName}\`` : "-");
         }
-        if (resource.containerName) {
-          details.push(`Container: \`${resource.containerName}\``);
+        if (hasContainer) {
+          cells.push(
+            resource.containerName ? `\`${resource.containerName}\`` : "-",
+          );
         }
-        if (resource.nodeName) {
-          details.push(`Node: \`${resource.nodeName}\``);
+        if (hasNode) {
+          cells.push(resource.nodeName ? `\`${resource.nodeName}\`` : "-");
         }
 
-        details.push(`Value: **${resource.metricValue}**`);
+        cells.push(`**${resource.metricValue}**`);
 
-        resourceLines.push(`- ${details.join(" | ")}`);
+        resourceLines.push(`| ${cells.join(" | ")} |`);
       }
 
       if (sortedResources.length > 10) {
         resourceLines.push(
-          `- ... and ${sortedResources.length - 10} more affected resources`,
+          `\n*... and ${sortedResources.length - 10} more affected resources*`,
         );
       }
 
       sections.push(
-        `\n\n**Affected Resources** (${sortedResources.length} total)\n${resourceLines.join("\n")}`,
+        `\n\n**Affected Resources** (${sortedResources.length} total)\n\n${resourceLines.join("\n")}`,
       );
 
       // Add root cause analysis based on metric type
@@ -770,6 +843,70 @@ ${contextBlock}
       if (analysis) {
         sections.push(`\n\n**Root Cause Analysis**\n${analysis}`);
       }
+
+      // Fetch recent container logs for the top affected resource during CrashLoopBackOff
+      if (
+        (breakdown.metricName === "k8s.container.restarts" ||
+          breakdown.metricName.includes("restart")) &&
+        input.monitor.projectId
+      ) {
+        const topResource: KubernetesAffectedResource = resourcesToShow[0]!;
+
+        try {
+          const logAttributes: Record<string, string> = {};
+
+          if (breakdown.clusterName) {
+            logAttributes["resource.k8s.cluster.name"] = breakdown.clusterName;
+          }
+
+          if (topResource.podName) {
+            logAttributes["resource.k8s.pod.name"] = topResource.podName;
+          }
+
+          if (topResource.containerName) {
+            logAttributes["resource.k8s.container.name"] =
+              topResource.containerName;
+          }
+
+          if (topResource.namespace) {
+            logAttributes["resource.k8s.namespace.name"] =
+              topResource.namespace;
+          }
+
+          const now: Date = OneUptimeDate.getCurrentDate();
+          const fifteenMinutesAgo: Date = OneUptimeDate.addRemoveMinutes(
+            now,
+            -15,
+          );
+
+          const logs: Array<JSONObject> =
+            await LogAggregationService.getExportLogs({
+              projectId: input.monitor.projectId,
+              startTime: fifteenMinutesAgo,
+              endTime: now,
+              limit: 50,
+              attributes: logAttributes,
+            });
+
+          if (logs.length > 0) {
+            const logLines: Array<string> = logs.map((log: JSONObject) => {
+              const timestamp: string = log["time"] ? String(log["time"]) : "";
+              const severity: string = log["severityText"]
+                ? String(log["severityText"])
+                : "INFO";
+              const body: string = log["body"] ? String(log["body"]) : "";
+              return `\`${timestamp}\` **${severity}** ${body}`;
+            });
+
+            sections.push(
+              `\n\n**Recent Container Logs** (${topResource.podName || "unknown pod"} / ${topResource.containerName || "unknown container"}, last 15 minutes)\n\n${logLines.join("\n\n")}`,
+            );
+          }
+        } catch (err) {
+          logger.error("Failed to fetch container logs for root cause context");
+          logger.error(err);
+        }
+      }
     }
 
     return sections.join("\n");

package/Server/Utils/Monitor/MonitorMetricUtil.ts

@@ -13,6 +13,7 @@ import MetricType from "../../../Models/DatabaseModels/MetricType";
 import BasicInfrastructureMetrics from "../../../Types/Infrastructure/BasicMetrics";
 import Dictionary from "../../../Types/Dictionary";
 import { JSONObject } from "../../../Types/JSON";
+import CapturedMetric from "../../../Types/Monitor/CustomCodeMonitor/CapturedMetric";
 import MonitorMetricType from "../../../Types/Monitor/MonitorMetricType";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import ServerMonitorResponse from "../../../Types/Monitor/ServerMonitor/ServerMonitorResponse";
@@ -533,6 +534,97 @@ export default class MonitorMetricUtil {
         metricType;
     }
 
+    // Process custom metrics from Custom Code and Synthetic Monitor responses
+    const customCodeMetrics: CapturedMetric[] =
+      (data.dataToProcess as ProbeMonitorResponse).customCodeMonitorResponse
+        ?.capturedMetrics || [];
+
+    const syntheticCustomMetrics: CapturedMetric[] = [];
+    const syntheticResponsesForMetrics: Array<SyntheticMonitorResponse> =
+      (data.dataToProcess as ProbeMonitorResponse).syntheticMonitorResponse ||
+      [];
+    for (const resp of syntheticResponsesForMetrics) {
+      if (resp.capturedMetrics) {
+        syntheticCustomMetrics.push(...resp.capturedMetrics);
+      }
+    }
+
+    const allCustomMetrics: CapturedMetric[] = [
+      ...customCodeMetrics,
+      ...syntheticCustomMetrics,
+    ].slice(0, 100);
+
+    if (allCustomMetrics.length > 0) {
+      logger.debug(
+        `${data.monitorId.toString()} - Processing ${allCustomMetrics.length} custom metrics`,
+      );
+    }
+
+    const reservedAttributeKeys: Set<string> = new Set([
+      "monitorId",
+      "projectId",
+      "monitorName",
+      "probeName",
+      "probeId",
+    ]);
+
+    for (const customMetric of allCustomMetrics) {
+      if (
+        !customMetric.name ||
+        typeof customMetric.name !== "string" ||
+        typeof customMetric.value !== "number" ||
+        isNaN(customMetric.value)
+      ) {
+        continue;
+      }
+
+      const prefixedName: string = `custom.monitor.${customMetric.name}`;
+
+      const extraAttributes: JSONObject = {
+        isCustomMetric: "true",
+      };
+
+      if ((data.dataToProcess as ProbeMonitorResponse).probeId) {
+        extraAttributes["probeId"] = (
+          data.dataToProcess as ProbeMonitorResponse
+        ).probeId.toString();
+      }
+
+      if (customMetric.attributes) {
+        for (const [key, val] of Object.entries(customMetric.attributes)) {
+          if (typeof val === "string" && !reservedAttributeKeys.has(key)) {
+            extraAttributes[key] = val;
+          }
+        }
+      }
+
+      const attributes: JSONObject = this.buildMonitorMetricAttributes({
+        monitorId: data.monitorId,
+        projectId: data.projectId,
+        monitorName: data.monitorName,
+        probeName: data.probeName,
+        extraAttributes: extraAttributes,
+      });
+
+      const metricRow: JSONObject = await this.buildMonitorMetricRow({
+        projectId: data.projectId,
+        monitorId: data.monitorId,
+        metricName: prefixedName,
+        value: customMetric.value,
+        attributes: attributes,
+        metricPointType: MetricPointType.Gauge,
+      });
+
+      metricRows.push(metricRow);
+
+      const metricType: MetricType = new MetricType();
+      metricType.name = prefixedName;
+      metricType.description = `Custom metric: ${customMetric.name}`;
+      metricType.unit = "";
+
+      metricNameServiceNameMap[prefixedName] = metricType;
+    }
+
     if (metricRows.length > 0) {
       await MetricService.insertJsonRows(metricRows);
     }

package/Server/Utils/Profiling.ts

@@ -0,0 +1,101 @@
+import Pyroscope from "@pyroscope/nodejs";
+import { EnableProfiling } from "../EnvironmentConfig";
+import logger from "./Logger";
+
+export default class Profiling {
+  public static init(data: { serviceName: string }): void {
+    if (!EnableProfiling) {
+      return;
+    }
+
+    const serverAddress: string | undefined = this.getServerAddress();
+    const authToken: string | undefined = this.getAuthToken();
+
+    if (!serverAddress) {
+      logger.warn(
+        "Profiling enabled but OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT not configured. Skipping profiling initialization.",
+      );
+      return;
+    }
+
+    try {
+      Pyroscope.init({
+        appName: data.serviceName,
+        serverAddress: serverAddress,
+        authToken: authToken,
+        wall: {
+          collectCpuTime: true,
+        },
+      });
+
+      Pyroscope.start();
+
+      logger.info(
+        `Profiling initialized for service: ${data.serviceName} -> ${serverAddress}`,
+      );
+    } catch (err) {
+      logger.error("Failed to initialize profiling:");
+      logger.error(err);
+    }
+
+    process.on("SIGTERM", () => {
+      Pyroscope.stop().catch((err: unknown) => {
+        logger.error("Error stopping profiler:");
+        logger.error(err);
+      });
+    });
+  }
+
+  private static getServerAddress(): string | undefined {
+    /*
+     * Use the OTLP endpoint base URL as the Pyroscope server address.
+     * The Pyroscope SDK will append /ingest to this URL.
+     * The final URL will be /pyroscope/ingest, routed by nginx to the telemetry service.
+     */
+    const endpoint: string | undefined =
+      process.env["OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT"];
+
+    if (!endpoint) {
+      return undefined;
+    }
+
+    /*
+     * Strip /otlp suffix if present and append /pyroscope
+     * The Pyroscope SDK appends /ingest, so the final URL will be /pyroscope/ingest
+     */
+    let baseUrl: string = endpoint;
+    if (baseUrl.endsWith("/otlp")) {
+      baseUrl = baseUrl.substring(0, baseUrl.length - 5);
+    }
+    if (baseUrl.endsWith("/")) {
+      baseUrl = baseUrl.substring(0, baseUrl.length - 1);
+    }
+
+    return `${baseUrl}/pyroscope`;
+  }
+
+  private static getAuthToken(): string | undefined {
+    /*
+     * Extract the OneUptime token from OTLP headers
+     * Format: "x-oneuptime-token=<value>;other-header=value"
+     */
+    const headersStr: string | undefined =
+      process.env["OPENTELEMETRY_EXPORTER_OTLP_HEADERS"];
+
+    if (!headersStr) {
+      return undefined;
+    }
+
+    const parts: Array<string> = headersStr.split(";");
+    for (const part of parts) {
+      const [key, value]: Array<string | undefined> = part.split("=") as Array<
+        string | undefined
+      >;
+      if (key === "x-oneuptime-token" && value) {
+        return value;
+      }
+    }
+
+    return undefined;
+  }
+}

package/Server/Utils/VM/VMRunner.ts

@@ -1,3 +1,4 @@
+import CapturedMetric from "../../../Types/Monitor/CustomCodeMonitor/CapturedMetric";
 import ReturnResult from "../../../Types/IsolatedVM/ReturnResult";
 import { JSONObject, JSONValue } from "../../../Types/JSON";
 import axios, { AxiosResponse } from "axios";
@@ -310,6 +311,9 @@ export default class VMRunner {
     const MAX_LOG_BYTES: number = 1_000_000; // 1MB cap
     let totalLogBytes: number = 0;
 
+    const capturedMetrics: CapturedMetric[] = [];
+    const MAX_METRICS: number = 100;
+
     // Track timer handles so we can clean them up after execution
     type TimerHandle = ReturnType<typeof setTimeout>;
     const pendingTimeouts: TimerHandle[] = [];
@@ -398,6 +402,47 @@ export default class VMRunner {
       proxyCache,
     );
 
+    sandbox["oneuptime"] = createSandboxProxy(
+      {
+        captureMetric: (
+          name: unknown,
+          value: unknown,
+          attributes?: unknown,
+        ): void => {
+          if (typeof name !== "string" || name.length === 0) {
+            return;
+          }
+          if (typeof value !== "number" || isNaN(value)) {
+            return;
+          }
+          if (capturedMetrics.length >= MAX_METRICS) {
+            return;
+          }
+          const metric: CapturedMetric = {
+            name: name.substring(0, 200),
+            value: value,
+          };
+          if (attributes && typeof attributes === "object") {
+            const safeAttrs: JSONObject = {};
+            for (const [k, v] of Object.entries(
+              attributes as Record<string, unknown>,
+            )) {
+              if (
+                typeof v === "string" ||
+                typeof v === "number" ||
+                typeof v === "boolean"
+              ) {
+                safeAttrs[k] = String(v);
+              }
+            }
+            metric.attributes = safeAttrs;
+          }
+          capturedMetrics.push(metric);
+        },
+      },
+      proxyCache,
+    );
+
     // Wrap any additional context (e.g. Playwright browser/page objects)
     if (options.context) {
       for (const key of Object.keys(options.context)) {
@@ -450,6 +495,7 @@ export default class VMRunner {
       return {
         returnValue: deepUnwrapProxies(returnVal),
         logMessages,
+        capturedMetrics,
       };
     } finally {
       // Clean up any lingering timers to prevent resource leaks
@@ -474,6 +520,8 @@ export default class VMRunner {
     const timeout: number = options.timeout || 5000;
 
     const logMessages: string[] = [];
+    const capturedMetrics: CapturedMetric[] = [];
+    const MAX_METRICS: number = 100;
 
     const isolate: ivm.Isolate = new ivm.Isolate({ memoryLimit: 128 });
 
@@ -499,6 +547,45 @@ export default class VMRunner {
         }))};
       `);
 
+      // oneuptime.captureMetric - fire-and-forget callback
+      await jail.set(
+        "_captureMetric",
+        new ivm.Callback(
+          (name: string, value: string, attributesJson?: string) => {
+            if (capturedMetrics.length >= MAX_METRICS) {
+              return;
+            }
+            const numValue: number = Number(value);
+            if (isNaN(numValue)) {
+              return;
+            }
+            const metric: CapturedMetric = {
+              name: String(name).substring(0, 200),
+              value: numValue,
+            };
+            if (attributesJson) {
+              try {
+                metric.attributes = JSON.parse(attributesJson) as JSONObject;
+              } catch {
+                // ignore invalid JSON
+              }
+            }
+            capturedMetrics.push(metric);
+          },
+        ),
+      );
+
+      await context.eval(`
+        const oneuptime = {
+          captureMetric: (name, value, attributes) => {
+            if (typeof name !== 'string' || name.length === 0) return;
+            if (typeof value !== 'number' || isNaN(value)) return;
+            const attrJson = attributes ? JSON.stringify(attributes) : undefined;
+            _captureMetric(String(name), String(value), attrJson);
+          }
+        };
+      `);
+
       // args - deep copy into isolate
       if (options.args) {
         await jail.set("_args", new ivm.ExternalCopy(options.args).copyInto());
@@ -961,6 +1048,7 @@ export default class VMRunner {
       return {
         returnValue,
         logMessages,
+        capturedMetrics,
       };
     } finally {
       if (!isolate.isDisposed) {