@oneuptime/common 10.5.17 → 10.5.18

package/Server/Utils/Telemetry/LogExceptionExtractor.ts

@@ -0,0 +1,289 @@
+import { JSONObject, JSONValue } from "../../../Types/JSON";
+import StackTraceParser, { ParsedStackTrace } from "./StackTraceParser";
+
+/**
+ * Result of detecting an exception inside a single log record. Shaped to feed
+ * the same ExceptionInstance (ClickHouse) + TelemetryException (Postgres)
+ * sinks the trace span-event path uses, so log-derived and span-derived
+ * exceptions group under one fingerprint when identical.
+ */
+export interface ExtractedLogException {
+  message: string;
+  exceptionType: string;
+  stackTrace: string;
+  parsedFrames: string; // JSON.stringify(StackFrame[]) or "[]"
+  escaped: boolean | null; // Path A may carry exception.escaped; Path B => null (unknown)
+}
+
+export interface LogExceptionExtractorInput {
+  body: string; // post-scrub log body
+  attributes: JSONObject; // post-scrub merged log attributes
+  severityNumber: number;
+  /**
+   * True when the log carries BOTH a traceId and a spanId — i.e. it was
+   * emitted inside an instrumented span. The span-exception path is the
+   * canonical source for those, so Path B (body scan) is suppressed to avoid
+   * double-counting (which would also inflate occuranceCount and the windowed
+   * exception monitor). Path A (explicit exception.* attributes) is NOT
+   * suppressed — those are an intentional structured exception record.
+   */
+  hasTraceAndSpan: boolean;
+}
+
+/**
+ * OTel log severityNumber >= 17 is ERROR (17-20) or FATAL (21-24). Path B only
+ * scans those — the overwhelming majority of logs are below this and never
+ * reach the parser.
+ */
+const MIN_ERROR_SEVERITY_NUMBER: number = 17;
+
+/**
+ * Only the first 16 KB of a body is parsed. A clean single-record stack trace
+ * fits comfortably (~150 frames); the top frames are the most diagnostic and
+ * are at the front. Guards against pathological multi-megabyte error logs on
+ * the hot path.
+ */
+const MAX_PARSE_BODY_LENGTH: number = 16 * 1024;
+
+/*
+ * Raw log bodies are unbounded (unlike SDK-bounded span exception.stacktrace),
+ * so clamp what we store into the ZSTD stackTrace column and the Postgres summary.
+ */
+const MAX_STORED_STACK_TRACE_LENGTH: number = 64 * 1024;
+const MAX_MESSAGE_LENGTH: number = 1024;
+
+/**
+ * Single pre-compiled signature for "this body plausibly contains a stack
+ * trace". Evaluated once before the (more expensive) multi-language parser.
+ * Covers: Python traceback header, JS/Java `at file:line`, Go panic/goroutine,
+ * Python `File "...", line N`, and a typed `SomethingException`/`SomethingError`.
+ */
+const LOOKS_LIKE_STACK_TRACE: RegExp =
+  /(?:Traceback \(most recent call last\)|\n\s+at\s+.+:\d+|\bpanic:\s|goroutine\s+\d+\s+\[|\n\s*File\s+"[^"]+",\s+line\s+\d+|\b[A-Za-z_][\w.$]*(?:Exception|Error)\b)/;
+
+// Header parsers for deriving exceptionType + message from a raw body.
+const PYTHON_TRACEBACK_HEADER: RegExp = /^Traceback \(most recent call last\):/;
+const JAVA_THREAD_PREFIX: RegExp = /^Exception in thread\s+"[^"]*"\s+(.*)$/;
+const GO_PANIC: RegExp = /^panic:\s*(.*)$/;
+/*
+ * Leading identifier is optional so a bare "Error: msg" / "Exception: msg"
+ * (common in Node.js) matches as well as "TypeError" / "java.lang.IOException".
+ */
+const TYPED_ERROR: RegExp =
+  /^((?:[A-Za-z_][\w.$]*)?(?:Error|Exception|Warning|Fault))(?::\s*([\s\S]*))?$/;
+const QUALIFIED_TYPE: RegExp = /^([A-Za-z_][\w.$]*\.[A-Za-z_][\w.$]*):\s*(.*)$/;
+
+export default class LogExceptionExtractor {
+  /**
+   * Detect an exception in a single log record. Returns null when none is
+   * found. Never throws — extraction must never fail log ingest.
+   */
+  public static extractFromLogRecord(
+    input: LogExceptionExtractorInput,
+  ): ExtractedLogException | null {
+    try {
+      /*
+       * Path A — explicit OTel exception.* attributes. Always on, cheapest,
+       * highest-signal (the app explicitly recorded an exception on the log).
+       */
+      const fromAttributes: ExtractedLogException | null =
+        LogExceptionExtractor.extractFromAttributes(input.attributes);
+      if (fromAttributes) {
+        return fromAttributes;
+      }
+
+      // Path B — raw body scan. Gated to keep the hot path cheap.
+      if (input.severityNumber < MIN_ERROR_SEVERITY_NUMBER) {
+        return null;
+      }
+      if (input.hasTraceAndSpan) {
+        return null;
+      }
+      return LogExceptionExtractor.extractFromBody(input.body);
+    } catch {
+      return null;
+    }
+  }
+
+  private static extractFromAttributes(
+    attributes: JSONObject,
+  ): ExtractedLogException | null {
+    if (!attributes) {
+      return null;
+    }
+
+    const stackTrace: string = asString(attributes["exception.stacktrace"]);
+    const exceptionType: string = asString(attributes["exception.type"]);
+    const message: string = asString(attributes["exception.message"]);
+
+    if (!stackTrace && !exceptionType && !message) {
+      return null;
+    }
+
+    const clampedStack: string = clamp(
+      stackTrace,
+      MAX_STORED_STACK_TRACE_LENGTH,
+    );
+
+    let parsedFrames: string = "[]";
+    if (clampedStack) {
+      try {
+        const parsed: ParsedStackTrace = StackTraceParser.parse(clampedStack);
+        parsedFrames = JSON.stringify(parsed.frames);
+      } catch {
+        parsedFrames = "[]";
+      }
+    }
+
+    return {
+      message: clamp(message, MAX_MESSAGE_LENGTH),
+      exceptionType: exceptionType,
+      stackTrace: clampedStack,
+      parsedFrames: parsedFrames,
+      escaped: toNullableBoolean(attributes["exception.escaped"]),
+    };
+  }
+
+  private static extractFromBody(body: string): ExtractedLogException | null {
+    if (!body) {
+      return null;
+    }
+
+    const sliced: string =
+      body.length > MAX_PARSE_BODY_LENGTH
+        ? body.slice(0, MAX_PARSE_BODY_LENGTH)
+        : body;
+
+    if (!LOOKS_LIKE_STACK_TRACE.test(sliced)) {
+      return null;
+    }
+
+    const parsed: ParsedStackTrace = StackTraceParser.parse(sliced);
+
+    /*
+     * Require at least one parsed frame. A signature match with zero frames is
+     * usually prose that merely mentions "...Error:" / "...Exception" — not an
+     * actual stack trace. Path A is the path allowed to emit without frames.
+     */
+    if (!parsed.frames || parsed.frames.length === 0) {
+      return null;
+    }
+
+    const header: { exceptionType: string; message: string } =
+      LogExceptionExtractor.parseHeader(sliced);
+
+    return {
+      message: clamp(header.message, MAX_MESSAGE_LENGTH),
+      exceptionType: header.exceptionType,
+      stackTrace: clamp(sliced, MAX_STORED_STACK_TRACE_LENGTH),
+      parsedFrames: JSON.stringify(parsed.frames),
+      escaped: null,
+    };
+  }
+
+  /**
+   * Best-effort derivation of exceptionType + message from a raw stack-trace
+   * body. A stable, clean exceptionType improves grouping and exception-monitor
+   * targeting, but a miss is harmless — the fingerprint also uses the message
+   * and the normalized stack trace.
+   */
+  private static parseHeader(body: string): {
+    exceptionType: string;
+    message: string;
+  } {
+    const lines: Array<string> = body
+      .split("\n")
+      .map((l: string) => {
+        return l.trim();
+      })
+      .filter((l: string) => {
+        return l.length > 0;
+      });
+
+    if (lines.length === 0) {
+      return { exceptionType: "", message: "" };
+    }
+
+    /*
+     * Python: header is "Traceback (most recent call last):"; the "Type: message"
+     * line is at the BOTTOM of the traceback, so scan upward for it.
+     */
+    if (PYTHON_TRACEBACK_HEADER.test(lines[0]!)) {
+      for (let i: number = lines.length - 1; i >= 0; i--) {
+        const candidate: RegExpMatchArray | null = lines[i]!.match(TYPED_ERROR);
+        if (candidate) {
+          return {
+            exceptionType: candidate[1] || "",
+            message: (candidate[2] || "").trim(),
+          };
+        }
+      }
+      // Truncated traceback with no type line — keep the last line as message.
+      return { exceptionType: "", message: lines[lines.length - 1]! };
+    }
+
+    // Strip Java's "Exception in thread "name"" prefix if present.
+    let firstLine: string = lines[0]!;
+    const threadMatch: RegExpMatchArray | null =
+      firstLine.match(JAVA_THREAD_PREFIX);
+    if (threadMatch) {
+      firstLine = threadMatch[1]!.trim();
+    }
+
+    const goMatch: RegExpMatchArray | null = firstLine.match(GO_PANIC);
+    if (goMatch) {
+      return { exceptionType: "panic", message: (goMatch[1] || "").trim() };
+    }
+
+    // "TypeError: msg", "java.lang.NullPointerException: msg", "...Exception"
+    const typedMatch: RegExpMatchArray | null = firstLine.match(TYPED_ERROR);
+    if (typedMatch) {
+      return {
+        exceptionType: typedMatch[1] || "",
+        message: (typedMatch[2] || "").trim(),
+      };
+    }
+
+    // Generic qualified "pkg.Sub.Type: msg"
+    const qualifiedMatch: RegExpMatchArray | null =
+      firstLine.match(QUALIFIED_TYPE);
+    if (qualifiedMatch) {
+      return {
+        exceptionType: qualifiedMatch[1] || "",
+        message: (qualifiedMatch[2] || "").trim(),
+      };
+    }
+
+    return { exceptionType: "", message: firstLine };
+  }
+}
+
+function asString(value: JSONValue | undefined): string {
+  if (typeof value === "string") {
+    return value;
+  }
+  return "";
+}
+
+function toNullableBoolean(value: JSONValue | undefined): boolean | null {
+  if (typeof value === "boolean") {
+    return value;
+  }
+  if (typeof value === "string") {
+    const normalized: string = value.trim().toLowerCase();
+    if (normalized === "true") {
+      return true;
+    }
+    if (normalized === "false") {
+      return false;
+    }
+  }
+  return null;
+}
+
+function clamp(value: string, max: number): string {
+  if (value && value.length > max) {
+    return value.slice(0, max);
+  }
+  return value;
+}

package/Server/Utils/Telemetry/StackTraceParser.ts

@@ -0,0 +1,423 @@
+/**
+ * Stack trace parser that transforms raw stack trace strings into structured frames.
+ * Supports JavaScript/Node.js, Python, Java, Go, Ruby, C#/.NET, and PHP stack traces.
+ */
+
+export interface StackFrame {
+  functionName: string;
+  fileName: string;
+  lineNumber: number;
+  columnNumber?: number;
+  inApp: boolean; // true if user code, false if library/framework code
+}
+
+export interface ParsedStackTrace {
+  frames: StackFrame[];
+  raw: string; // original raw stack trace string
+}
+
+// Known library/framework path patterns that indicate non-app code
+const LIBRARY_PATTERNS: Array<RegExp> = [
+  // Node.js internals
+  /^node:/,
+  /^internal\//,
+  /node_modules\//,
+  /^events\.js$/,
+  /^timers\.js$/,
+  /^util\.js$/,
+  /^net\.js$/,
+  /^stream\.js$/,
+  /^buffer\.js$/,
+  // Python
+  /\/site-packages\//,
+  /\/dist-packages\//,
+  /\/lib\/python\d+\.\d+\//,
+  /\/usr\/lib\//,
+  /\/usr\/local\/lib\//,
+  /\/venv\//,
+  /\/\.venv\//,
+  /\/virtualenv\//,
+  // Java
+  /^java\./,
+  /^javax\./,
+  /^sun\./,
+  /^com\.sun\./,
+  /^org\.springframework\./,
+  /^org\.apache\./,
+  /^org\.hibernate\./,
+  /^org\.eclipse\./,
+  /^io\.netty\./,
+  /^com\.google\./,
+  /^org\.junit\./,
+  // Go
+  /^runtime\//,
+  /^net\/http\//,
+  /^testing\//,
+  /\/vendor\//,
+  /\/pkg\/mod\//,
+  // Ruby
+  /\/gems\//,
+  /\/rubygems\//,
+  /\/ruby\/\d+\.\d+\.\d+\//,
+  // C#/.NET
+  /^System\./,
+  /^Microsoft\./,
+  /^Newtonsoft\./,
+  // PHP
+  /\/vendor\//,
+  /^phar:\/\//,
+];
+
+export default class StackTraceParser {
+  /**
+   * Parse a raw stack trace string into structured frames.
+   * Auto-detects the language and applies the appropriate parser.
+   */
+  public static parse(rawStackTrace: string): ParsedStackTrace {
+    if (!rawStackTrace || rawStackTrace.trim().length === 0) {
+      return { frames: [], raw: rawStackTrace || "" };
+    }
+
+    const lines: string[] = rawStackTrace.split("\n").map((l: string) => {
+      return l.trim();
+    });
+
+    // Try each parser and use the one that produces the most frames
+    const parsers: Array<(lines: string[]) => StackFrame[]> = [
+      StackTraceParser.parseJavaScript,
+      StackTraceParser.parsePython,
+      StackTraceParser.parseJava,
+      StackTraceParser.parseGo,
+      StackTraceParser.parseRuby,
+      StackTraceParser.parseCSharp,
+      StackTraceParser.parsePHP,
+    ];
+
+    let bestFrames: StackFrame[] = [];
+
+    for (const parser of parsers) {
+      try {
+        const frames: StackFrame[] = parser(lines);
+        if (frames.length > bestFrames.length) {
+          bestFrames = frames;
+        }
+      } catch {
+        // Skip failing parsers
+      }
+    }
+
+    return {
+      frames: bestFrames,
+      raw: rawStackTrace,
+    };
+  }
+
+  /**
+   * Determine if a file path is application code (not library/framework).
+   */
+  private static isAppCode(filePath: string): boolean {
+    if (!filePath) {
+      return true;
+    }
+
+    for (const pattern of LIBRARY_PATTERNS) {
+      if (pattern.test(filePath)) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * Parse JavaScript/Node.js stack traces.
+   * Format: `at functionName (filePath:line:col)` or `at filePath:line:col`
+   */
+  private static parseJavaScript(lines: string[]): StackFrame[] {
+    const frames: StackFrame[] = [];
+
+    // Pattern 1: at functionName (filePath:line:col)
+    const patternWithParens: RegExp = /^at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)$/;
+    // Pattern 2: at filePath:line:col
+    const patternWithoutParens: RegExp = /^at\s+(.+?):(\d+):(\d+)$/;
+    // Pattern 3: at functionName (filePath:line)
+    const patternWithParensNoCol: RegExp = /^at\s+(.+?)\s+\((.+?):(\d+)\)$/;
+    // Pattern 4: at eval (eval at functionName (filePath:line:col))
+    const patternEval: RegExp =
+      /^at\s+eval\s+\(eval\s+at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/;
+
+    for (const line of lines) {
+      let match: RegExpMatchArray | null = null;
+
+      match = line.match(patternEval);
+      if (match) {
+        frames.push({
+          functionName: `eval at ${match[1]!}`,
+          fileName: match[2]!,
+          lineNumber: parseInt(match[3]!, 10),
+          columnNumber: parseInt(match[4]!, 10),
+          inApp: StackTraceParser.isAppCode(match[2]!),
+        });
+        continue;
+      }
+
+      match = line.match(patternWithParens);
+      if (match) {
+        frames.push({
+          functionName: match[1]!,
+          fileName: match[2]!,
+          lineNumber: parseInt(match[3]!, 10),
+          columnNumber: parseInt(match[4]!, 10),
+          inApp: StackTraceParser.isAppCode(match[2]!),
+        });
+        continue;
+      }
+
+      match = line.match(patternWithParensNoCol);
+      if (match) {
+        frames.push({
+          functionName: match[1]!,
+          fileName: match[2]!,
+          lineNumber: parseInt(match[3]!, 10),
+          inApp: StackTraceParser.isAppCode(match[2]!),
+        });
+        continue;
+      }
+
+      match = line.match(patternWithoutParens);
+      if (match) {
+        frames.push({
+          functionName: "<anonymous>",
+          fileName: match[1]!,
+          lineNumber: parseInt(match[2]!, 10),
+          columnNumber: parseInt(match[3]!, 10),
+          inApp: StackTraceParser.isAppCode(match[1]!),
+        });
+        continue;
+      }
+    }
+
+    return frames;
+  }
+
+  /**
+   * Parse Python stack traces.
+   * Format: `File "path", line N, in function`
+   */
+  private static parsePython(lines: string[]): StackFrame[] {
+    const frames: StackFrame[] = [];
+    const pattern: RegExp =
+      /^File\s+"(.+?)",\s+line\s+(\d+)(?:,\s+in\s+(.+))?$/;
+
+    for (const line of lines) {
+      const match: RegExpMatchArray | null = line.match(pattern);
+      if (match) {
+        frames.push({
+          functionName: match[3] || "<module>",
+          fileName: match[1]!,
+          lineNumber: parseInt(match[2]!, 10),
+          inApp: StackTraceParser.isAppCode(match[1]!),
+        });
+      }
+    }
+
+    return frames;
+  }
+
+  /**
+   * Parse Java stack traces.
+   * Format: `at package.Class.method(File.java:line)`
+   */
+  private static parseJava(lines: string[]): StackFrame[] {
+    const frames: StackFrame[] = [];
+    // Pattern: at com.package.Class.method(File.java:123)
+    const pattern: RegExp = /^at\s+([\w.$]+)\(([\w.]+):(\d+)\)$/;
+    // Pattern for native methods: at com.package.Class.method(Native Method)
+    const patternNative: RegExp = /^at\s+([\w.$]+)\(Native Method\)$/;
+    // Pattern for unknown source: at com.package.Class.method(Unknown Source)
+    const patternUnknown: RegExp = /^at\s+([\w.$]+)\(Unknown Source\)$/;
+
+    for (const line of lines) {
+      let match: RegExpMatchArray | null = null;
+
+      match = line.match(pattern);
+      if (match) {
+        const fullMethod: string = match[1]!;
+        frames.push({
+          functionName: fullMethod,
+          fileName: match[2]!,
+          lineNumber: parseInt(match[3]!, 10),
+          inApp: StackTraceParser.isAppCode(fullMethod),
+        });
+        continue;
+      }
+
+      match = line.match(patternNative);
+      if (match) {
+        frames.push({
+          functionName: match[1]!,
+          fileName: "Native Method",
+          lineNumber: 0,
+          inApp: false,
+        });
+        continue;
+      }
+
+      match = line.match(patternUnknown);
+      if (match) {
+        frames.push({
+          functionName: match[1]!,
+          fileName: "Unknown Source",
+          lineNumber: 0,
+          inApp: StackTraceParser.isAppCode(match[1]!),
+        });
+        continue;
+      }
+    }
+
+    return frames;
+  }
+
+  /**
+   * Parse Go stack traces.
+   * Format: `package/file.go:line +0xNN` or `goroutine N [reason]:`
+   * Go stack traces have pairs of lines:
+   *   functionName(args)
+   *   /path/to/file.go:line +0xNN
+   */
+  private static parseGo(lines: string[]): StackFrame[] {
+    const frames: StackFrame[] = [];
+    const filePattern: RegExp = /^(.+\.go):(\d+)\s*(?:\+0x[0-9a-f]+)?$/;
+
+    for (let i: number = 0; i < lines.length; i++) {
+      const line: string = lines[i]!;
+
+      // Skip goroutine headers
+      if (line.startsWith("goroutine ")) {
+        continue;
+      }
+
+      // Look for file:line pattern
+      const match: RegExpMatchArray | null = line.match(filePattern);
+      if (match) {
+        // The previous line should be the function name
+        let functionName: string = "<unknown>";
+        if (i > 0 && lines[i - 1]) {
+          // Remove arguments from function name
+          const funcLine: string = lines[i - 1]!;
+          const parenIndex: number = funcLine.indexOf("(");
+          functionName =
+            parenIndex > 0 ? funcLine.substring(0, parenIndex) : funcLine;
+        }
+
+        frames.push({
+          functionName: functionName,
+          fileName: match[1]!,
+          lineNumber: parseInt(match[2]!, 10),
+          inApp: StackTraceParser.isAppCode(match[1]!),
+        });
+      }
+    }
+
+    return frames;
+  }
+
+  /**
+   * Parse Ruby stack traces.
+   * Format: `file:line:in 'method'` or `file:line:in \`method'`
+   */
+  private static parseRuby(lines: string[]): StackFrame[] {
+    const frames: StackFrame[] = [];
+    const pattern: RegExp = /^(.+?):(\d+):in\s+[`'](.+?)'$/;
+
+    for (const line of lines) {
+      const match: RegExpMatchArray | null = line.match(pattern);
+      if (match) {
+        frames.push({
+          functionName: match[3]!,
+          fileName: match[1]!,
+          lineNumber: parseInt(match[2]!, 10),
+          inApp: StackTraceParser.isAppCode(match[1]!),
+        });
+      }
+    }
+
+    return frames;
+  }
+
+  /**
+   * Parse C#/.NET stack traces.
+   * Format: `at Namespace.Class.Method(params) in file:line N`
+   */
+  private static parseCSharp(lines: string[]): StackFrame[] {
+    const frames: StackFrame[] = [];
+    // Pattern: at Namespace.Class.Method(params) in /path/to/file.cs:line 42
+    const patternWithFile: RegExp = /^at\s+(.+?)\s+in\s+(.+?):line\s+(\d+)$/;
+    // Pattern: at Namespace.Class.Method(params)
+    const patternWithoutFile: RegExp = /^at\s+([\w.<>+]+\(.*?\))$/;
+
+    for (const line of lines) {
+      let match: RegExpMatchArray | null = null;
+
+      match = line.match(patternWithFile);
+      if (match) {
+        frames.push({
+          functionName: match[1]!,
+          fileName: match[2]!,
+          lineNumber: parseInt(match[3]!, 10),
+          inApp: StackTraceParser.isAppCode(match[1]!),
+        });
+        continue;
+      }
+
+      match = line.match(patternWithoutFile);
+      if (match) {
+        frames.push({
+          functionName: match[1]!,
+          fileName: "",
+          lineNumber: 0,
+          inApp: StackTraceParser.isAppCode(match[1]!),
+        });
+        continue;
+      }
+    }
+
+    return frames;
+  }
+
+  /**
+   * Parse PHP stack traces.
+   * Format: `#N /path/to/file.php(line): Class->method()`
+   */
+  private static parsePHP(lines: string[]): StackFrame[] {
+    const frames: StackFrame[] = [];
+    // Pattern: #0 /path/to/file.php(42): ClassName->method()
+    const pattern: RegExp = /^#\d+\s+(.+?)\((\d+)\):\s+(.+)$/;
+    // Pattern: #0 {main}
+    const patternMain: RegExp = /^#\d+\s+\{main\}$/;
+
+    for (const line of lines) {
+      if (patternMain.test(line)) {
+        frames.push({
+          functionName: "{main}",
+          fileName: "",
+          lineNumber: 0,
+          inApp: true,
+        });
+        continue;
+      }
+
+      const match: RegExpMatchArray | null = line.match(pattern);
+      if (match) {
+        frames.push({
+          functionName: match[3]!.replace(/\(\)$/, ""),
+          fileName: match[1]!,
+          lineNumber: parseInt(match[2]!, 10),
+          inApp: StackTraceParser.isAppCode(match[1]!),
+        });
+      }
+    }
+
+    return frames;
+  }
+}