@oneuptime/common 10.5.9 → 10.5.18

package/Server/Utils/GracefulShutdown.ts

@@ -0,0 +1,194 @@
+import logger from "./Logger";
+
+/*
+ * Centralized graceful-shutdown coordinator.
+ *
+ * Before this existed, each subsystem registered its own
+ * process.on("SIGTERM" | "SIGINT") handler independently. The telemetry
+ * handler in particular called process.exit(0) as soon as the OTEL SDK
+ * flushed, which raced every other handler and — crucially — meant the
+ * Postgres / Redis / Clickhouse pools were never drained. Connections were
+ * left for the OS socket teardown to reap (and leaked outright on SIGKILL or
+ * a network partition).
+ *
+ * Now every subsystem registers an async cleanup callback here, and this class
+ * is the single owner of the signal handlers and of process.exit. Handlers run
+ * in ascending priority order (lower first) so we stop accepting new work
+ * before tearing down the resources that work depends on:
+ *
+ *   HttpServer (10) -> stop accepting new HTTP requests
+ *   Workers    (20) -> stop pulling new queue jobs, finish in-flight jobs
+ *   Buffers    (30) -> flush in-memory write buffers to their datastore
+ *   DataStores (40) -> drain Postgres / Redis / Clickhouse pools
+ *   Telemetry  (50) -> flush traces / metrics / logs / profiles last
+ *
+ * Handlers in the same tier run concurrently. Each handler is bounded by a
+ * per-handler timeout, and the whole sequence by an overall deadline, so a
+ * single hung handler can never wedge the shutdown.
+ */
+export enum ShutdownPriority {
+  HttpServer = 10,
+  Workers = 20,
+  Buffers = 30,
+  DataStores = 40,
+  Telemetry = 50,
+}
+
+export type ShutdownCallback = () => Promise<void> | void;
+
+interface RegisteredShutdownHandler {
+  name: string;
+  priority: ShutdownPriority;
+  callback: ShutdownCallback;
+}
+
+export default class GracefulShutdown {
+  private static handlers: Array<RegisteredShutdownHandler> = [];
+  private static signalListenersInstalled: boolean = false;
+  private static isShuttingDown: boolean = false;
+
+  /*
+   * How long a single handler may run before we give up on it and move on.
+   * Kept comfortably under the orchestrator (Kubernetes) default
+   * terminationGracePeriodSeconds of 30s.
+   */
+  private static readonly perHandlerTimeoutMs: number = 10_000;
+
+  /*
+   * Hard ceiling for the entire shutdown. If we blow past this we force-exit
+   * rather than risk being SIGKILLed mid-cleanup.
+   */
+  private static readonly overallTimeoutMs: number = 25_000;
+
+  /*
+   * Register a cleanup callback to run on SIGTERM / SIGINT. Registering by a
+   * stable name is idempotent: a repeat registration (e.g. a second connect())
+   * replaces the previous callback instead of stacking a duplicate. Callers
+   * that own multiple independent resources (e.g. two Clickhouse pools) must
+   * therefore pass distinct names.
+   */
+  public static registerHandler(
+    name: string,
+    priority: ShutdownPriority,
+    callback: ShutdownCallback,
+  ): void {
+    const existingIndex: number = this.handlers.findIndex(
+      (handler: RegisteredShutdownHandler) => {
+        return handler.name === name;
+      },
+    );
+
+    if (existingIndex >= 0) {
+      this.handlers[existingIndex] = { name, priority, callback };
+    } else {
+      this.handlers.push({ name, priority, callback });
+    }
+
+    this.installSignalListeners();
+  }
+
+  private static installSignalListeners(): void {
+    if (this.signalListenersInstalled) {
+      return;
+    }
+    this.signalListenersInstalled = true;
+
+    process.on("SIGTERM", () => {
+      void this.shutdown("SIGTERM");
+    });
+    process.on("SIGINT", () => {
+      void this.shutdown("SIGINT");
+    });
+  }
+
+  public static async shutdown(signal: string): Promise<void> {
+    if (this.isShuttingDown) {
+      /*
+       * A second signal while we're already draining means the operator (or
+       * orchestrator) is impatient. Bail out immediately.
+       */
+      logger.warn(
+        `GracefulShutdown: received ${signal} while already shutting down. Forcing exit.`,
+      );
+      return process.exit(1);
+    }
+
+    this.isShuttingDown = true;
+    logger.info(
+      `GracefulShutdown: received ${signal}. Draining ${this.handlers.length} handler(s)...`,
+    );
+
+    const forceExitTimer: ReturnType<typeof setTimeout> = setTimeout(() => {
+      logger.error(
+        `GracefulShutdown: exceeded ${this.overallTimeoutMs}ms overall deadline. Forcing exit.`,
+      );
+      return process.exit(1);
+    }, this.overallTimeoutMs);
+
+    // Don't let this timer keep the event loop alive on its own.
+    forceExitTimer.unref();
+
+    // Run handlers tier by tier; lower priority tiers complete before the next.
+    const tiers: Array<number> = Array.from(
+      new Set(
+        this.handlers.map((handler: RegisteredShutdownHandler) => {
+          return handler.priority;
+        }),
+      ),
+    ).sort((a: number, b: number) => {
+      return a - b;
+    });
+
+    for (const tier of tiers) {
+      const handlersInTier: Array<RegisteredShutdownHandler> =
+        this.handlers.filter((handler: RegisteredShutdownHandler) => {
+          return handler.priority === tier;
+        });
+
+      // Handlers within a tier are independent, so run them concurrently.
+      await Promise.all(
+        handlersInTier.map((handler: RegisteredShutdownHandler) => {
+          return this.runHandlerWithTimeout(handler);
+        }),
+      );
+    }
+
+    clearTimeout(forceExitTimer);
+    logger.info("GracefulShutdown: all handlers complete. Exiting cleanly.");
+    return process.exit(0);
+  }
+
+  private static async runHandlerWithTimeout(
+    handler: RegisteredShutdownHandler,
+  ): Promise<void> {
+    logger.debug(`GracefulShutdown: running handler "${handler.name}"...`);
+
+    let timer: ReturnType<typeof setTimeout> | null = null;
+
+    const timeout: Promise<void> = new Promise<void>((resolve: () => void) => {
+      timer = setTimeout(() => {
+        logger.warn(
+          `GracefulShutdown: handler "${handler.name}" exceeded ${this.perHandlerTimeoutMs}ms. Moving on.`,
+        );
+        return resolve();
+      }, this.perHandlerTimeoutMs);
+      timer.unref();
+    });
+
+    const run: Promise<void> = (async (): Promise<void> => {
+      try {
+        await handler.callback();
+        logger.debug(`GracefulShutdown: handler "${handler.name}" done.`);
+      } catch (err) {
+        logger.error(`GracefulShutdown: handler "${handler.name}" failed:`);
+        logger.error(err);
+      }
+    })();
+
+    await Promise.race([run, timeout]);
+
+    if (timer) {
+      clearTimeout(timer);
+    }
+  }
+}

package/Server/Utils/Logger.ts

@@ -1,5 +1,6 @@
 import { LogLevel } from "../EnvironmentConfig";
 import OneUptimeTelemetry, { TelemetryLogger } from "./Telemetry";
+import TelemetryContext from "./Telemetry/TelemetryContext";
 import { SeverityNumber } from "@opentelemetry/api-logs";
 import Exception from "../../Types/Exception/Exception";
 import { JSONObject } from "../../Types/JSON";
@@ -171,9 +172,19 @@ export default class logger {
         return;
       }
 
+      /*
+       * Merge ambient TelemetryContext attributes (projectId, userId,
+       * monitorId, requestId, ...) into every log record. Attributes passed
+       * explicitly to the log call take precedence over the ambient context.
+       */
+      const mergedAttributes: LogAttributes = {
+        ...TelemetryContext.getAttributes(),
+        ...(data.attributes || {}),
+      };
+
       const sanitizedAttributes:
         | Record<string, string | number | boolean>
-        | undefined = this.sanitizeAttributes(data.attributes);
+        | undefined = this.sanitizeAttributes(mergedAttributes);
 
       logger.emit({
         body: this.serializeLogBody(data.body),

package/Server/Utils/Monitor/MonitorLogUtil.ts

@@ -2,6 +2,7 @@ import MonitorLogService from "../../Services/MonitorLogService";
 import GlobalConfigService from "../../Services/GlobalConfigService";
 import GlobalConfig from "../../../Models/DatabaseModels/GlobalConfig";
 import logger from "../Logger";
+import GracefulShutdown, { ShutdownPriority } from "../GracefulShutdown";
 import OneUptimeDate from "../../../Types/Date";
 import ObjectID from "../../../Types/ObjectID";
 import { JSONObject } from "../../../Types/JSON";
@@ -44,8 +45,9 @@ export default class MonitorLogUtil {
    * here until either MONITOR_LOG_FLUSH_BATCH_SIZE rows arrive
    * (size trigger) or MONITOR_LOG_FLUSH_INTERVAL_MS elapses since
    * the first row entered an empty buffer (time trigger),
-   * whichever comes first. On graceful shutdown the SIGTERM /
-   * SIGINT hook below drains the buffer before the process exits.
+   * whichever comes first. On graceful shutdown the registered
+   * GracefulShutdown handler below drains the buffer (in the
+   * Buffers tier, before the datastores are torn down).
    */
   private static buffer: Array<JSONObject> = [];
   private static flushTimer: NodeJS.Timeout | null = null;
@@ -233,10 +235,12 @@ export default class MonitorLogUtil {
   }
 
   /*
-   * Register SIGTERM / SIGINT handlers exactly once, lazily on
-   * first ingest. We avoid registering at module-load time so
-   * tooling that imports this file (e.g. migration runners,
-   * CLI scripts) doesn't end up with stray process listeners.
+   * Register the shutdown flush exactly once, lazily on first ingest. We avoid
+   * registering at module-load time so tooling that imports this file (e.g.
+   * migration runners, CLI scripts) doesn't end up holding a stray handler.
+   *
+   * Runs in the Buffers tier — ahead of the DataStores tier — so the buffer is
+   * drained to Clickhouse before the datastore pools are torn down.
    */
   private static ensureShutdownHooks(): void {
     if (this.shutdownHooksRegistered) {
@@ -244,16 +248,17 @@ export default class MonitorLogUtil {
     }
     this.shutdownHooksRegistered = true;
 
-    const flushOnShutdown: () => Promise<void> = async (): Promise<void> => {
-      try {
-        await this.flushAndWait();
-      } catch (err) {
-        logger.error("Error flushing MonitorLog buffer on shutdown:");
-        logger.error(err);
-      }
-    };
-
-    process.on("SIGTERM", flushOnShutdown);
-    process.on("SIGINT", flushOnShutdown);
+    GracefulShutdown.registerHandler(
+      "MonitorLogUtil",
+      ShutdownPriority.Buffers,
+      async (): Promise<void> => {
+        try {
+          await this.flushAndWait();
+        } catch (err) {
+          logger.error("Error flushing MonitorLog buffer on shutdown:");
+          logger.error(err);
+        }
+      },
+    );
   }
 }

package/Server/Utils/Profiling.ts

@@ -1,6 +1,7 @@
 import Pyroscope from "@pyroscope/nodejs";
 import { EnableProfiling } from "../EnvironmentConfig";
 import logger, { LogAttributes } from "./Logger";
+import GracefulShutdown, { ShutdownPriority } from "./GracefulShutdown";
 
 export default class Profiling {
   public static init(data: { serviceName: string }): void {
@@ -44,12 +45,19 @@ export default class Profiling {
       logger.error(err, profilingLogAttributes);
     }
 
-    process.on("SIGTERM", () => {
-      Pyroscope.stop().catch((err: unknown) => {
-        logger.error("Error stopping profiler:", profilingLogAttributes);
-        logger.error(err, profilingLogAttributes);
-      });
-    });
+    // Stop the profiler last (Telemetry tier), alongside the OTEL flush.
+    GracefulShutdown.registerHandler(
+      "Profiling",
+      ShutdownPriority.Telemetry,
+      async (): Promise<void> => {
+        try {
+          await Pyroscope.stop();
+        } catch (err) {
+          logger.error("Error stopping profiler:", profilingLogAttributes);
+          logger.error(err, profilingLogAttributes);
+        }
+      },
+    );
   }
 
   private static getServerAddress(): string | undefined {

package/Server/Utils/StartServer.ts

@@ -29,6 +29,7 @@ import logger, {
 import "./Process";
 import Response from "./Response";
 import SpanUtil from "./Telemetry/SpanUtil";
+import TelemetryContext from "./Telemetry/TelemetryContext";
 import { api } from "@opentelemetry/sdk-node";
 import StatusCode from "../../Types/API/StatusCode";
 import HTTPErrorResponse from "../../Types/API/HTTPErrorResponse";
@@ -213,12 +214,19 @@ app.use((req: ExpressRequest, _res: ExpressResponse, next: NextFunction) => {
   const requestId: string = crypto.randomUUID();
   (req as OneUptimeRequest).requestId = requestId;
 
-  // Tag the current span with requestId so all downstream spans inherit context
-  SpanUtil.addAttributesToCurrentSpan({
-    requestId: requestId,
-  });
+  /*
+   * Open a telemetry-context scope for the entire request. requestId is seeded
+   * here; projectId/userId are added later by the auth middleware. Because
+   * ContextSpanProcessor and Logger read this ambient context, every span and
+   * log produced downstream inherits it automatically.
+   */
+  TelemetryContext.runWithContext({ requestId: requestId }, () => {
+    SpanUtil.addAttributesToCurrentSpan({
+      requestId: requestId,
+    });
 
-  next();
+    next();
+  });
 });
 
 export interface InitFuctionOptions {

package/Server/Utils/Telemetry/ContextSpanProcessor.ts

@@ -0,0 +1,48 @@
+import TelemetryContext from "./TelemetryContext";
+import type { Context } from "@opentelemetry/api";
+import type {
+  ReadableSpan,
+  Span,
+  SpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+
+/**
+ * Copies the ambient {@link TelemetryContext} attributes (projectId, userId,
+ * monitorId, incidentId, requestId, ...) onto every span at creation time.
+ *
+ * Combined with `TelemetryContext` scopes seeded at each entry point (HTTP
+ * request, worker job, probe check, cron run), this makes the full
+ * tenant/business context queryable on all spans — including the ~1958
+ * attribute-less `@CaptureSpan` spans — without touching any of those call
+ * sites.
+ */
+export default class ContextSpanProcessor implements SpanProcessor {
+  public onStart(span: Span, _parentContext: Context): void {
+    try {
+      const attributes: Record<string, string | number | boolean> =
+        TelemetryContext.getAttributes();
+
+      for (const key in attributes) {
+        const value: string | number | boolean | undefined = attributes[key];
+
+        if (value !== undefined && value !== null) {
+          span.setAttribute(key, value);
+        }
+      }
+    } catch {
+      // Context enrichment must never break span creation.
+    }
+  }
+
+  public onEnd(_span: ReadableSpan): void {
+    // no-op: enrichment happens entirely in onStart.
+  }
+
+  public shutdown(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  public forceFlush(): Promise<void> {
+    return Promise.resolve();
+  }
+}

package/Server/Utils/Telemetry/LogExceptionExtractor.ts

@@ -0,0 +1,289 @@
+import { JSONObject, JSONValue } from "../../../Types/JSON";
+import StackTraceParser, { ParsedStackTrace } from "./StackTraceParser";
+
+/**
+ * Result of detecting an exception inside a single log record. Shaped to feed
+ * the same ExceptionInstance (ClickHouse) + TelemetryException (Postgres)
+ * sinks the trace span-event path uses, so log-derived and span-derived
+ * exceptions group under one fingerprint when identical.
+ */
+export interface ExtractedLogException {
+  message: string;
+  exceptionType: string;
+  stackTrace: string;
+  parsedFrames: string; // JSON.stringify(StackFrame[]) or "[]"
+  escaped: boolean | null; // Path A may carry exception.escaped; Path B => null (unknown)
+}
+
+export interface LogExceptionExtractorInput {
+  body: string; // post-scrub log body
+  attributes: JSONObject; // post-scrub merged log attributes
+  severityNumber: number;
+  /**
+   * True when the log carries BOTH a traceId and a spanId — i.e. it was
+   * emitted inside an instrumented span. The span-exception path is the
+   * canonical source for those, so Path B (body scan) is suppressed to avoid
+   * double-counting (which would also inflate occuranceCount and the windowed
+   * exception monitor). Path A (explicit exception.* attributes) is NOT
+   * suppressed — those are an intentional structured exception record.
+   */
+  hasTraceAndSpan: boolean;
+}
+
+/**
+ * OTel log severityNumber >= 17 is ERROR (17-20) or FATAL (21-24). Path B only
+ * scans those — the overwhelming majority of logs are below this and never
+ * reach the parser.
+ */
+const MIN_ERROR_SEVERITY_NUMBER: number = 17;
+
+/**
+ * Only the first 16 KB of a body is parsed. A clean single-record stack trace
+ * fits comfortably (~150 frames); the top frames are the most diagnostic and
+ * are at the front. Guards against pathological multi-megabyte error logs on
+ * the hot path.
+ */
+const MAX_PARSE_BODY_LENGTH: number = 16 * 1024;
+
+/*
+ * Raw log bodies are unbounded (unlike SDK-bounded span exception.stacktrace),
+ * so clamp what we store into the ZSTD stackTrace column and the Postgres summary.
+ */
+const MAX_STORED_STACK_TRACE_LENGTH: number = 64 * 1024;
+const MAX_MESSAGE_LENGTH: number = 1024;
+
+/**
+ * Single pre-compiled signature for "this body plausibly contains a stack
+ * trace". Evaluated once before the (more expensive) multi-language parser.
+ * Covers: Python traceback header, JS/Java `at file:line`, Go panic/goroutine,
+ * Python `File "...", line N`, and a typed `SomethingException`/`SomethingError`.
+ */
+const LOOKS_LIKE_STACK_TRACE: RegExp =
+  /(?:Traceback \(most recent call last\)|\n\s+at\s+.+:\d+|\bpanic:\s|goroutine\s+\d+\s+\[|\n\s*File\s+"[^"]+",\s+line\s+\d+|\b[A-Za-z_][\w.$]*(?:Exception|Error)\b)/;
+
+// Header parsers for deriving exceptionType + message from a raw body.
+const PYTHON_TRACEBACK_HEADER: RegExp = /^Traceback \(most recent call last\):/;
+const JAVA_THREAD_PREFIX: RegExp = /^Exception in thread\s+"[^"]*"\s+(.*)$/;
+const GO_PANIC: RegExp = /^panic:\s*(.*)$/;
+/*
+ * Leading identifier is optional so a bare "Error: msg" / "Exception: msg"
+ * (common in Node.js) matches as well as "TypeError" / "java.lang.IOException".
+ */
+const TYPED_ERROR: RegExp =
+  /^((?:[A-Za-z_][\w.$]*)?(?:Error|Exception|Warning|Fault))(?::\s*([\s\S]*))?$/;
+const QUALIFIED_TYPE: RegExp = /^([A-Za-z_][\w.$]*\.[A-Za-z_][\w.$]*):\s*(.*)$/;
+
+export default class LogExceptionExtractor {
+  /**
+   * Detect an exception in a single log record. Returns null when none is
+   * found. Never throws — extraction must never fail log ingest.
+   */
+  public static extractFromLogRecord(
+    input: LogExceptionExtractorInput,
+  ): ExtractedLogException | null {
+    try {
+      /*
+       * Path A — explicit OTel exception.* attributes. Always on, cheapest,
+       * highest-signal (the app explicitly recorded an exception on the log).
+       */
+      const fromAttributes: ExtractedLogException | null =
+        LogExceptionExtractor.extractFromAttributes(input.attributes);
+      if (fromAttributes) {
+        return fromAttributes;
+      }
+
+      // Path B — raw body scan. Gated to keep the hot path cheap.
+      if (input.severityNumber < MIN_ERROR_SEVERITY_NUMBER) {
+        return null;
+      }
+      if (input.hasTraceAndSpan) {
+        return null;
+      }
+      return LogExceptionExtractor.extractFromBody(input.body);
+    } catch {
+      return null;
+    }
+  }
+
+  private static extractFromAttributes(
+    attributes: JSONObject,
+  ): ExtractedLogException | null {
+    if (!attributes) {
+      return null;
+    }
+
+    const stackTrace: string = asString(attributes["exception.stacktrace"]);
+    const exceptionType: string = asString(attributes["exception.type"]);
+    const message: string = asString(attributes["exception.message"]);
+
+    if (!stackTrace && !exceptionType && !message) {
+      return null;
+    }
+
+    const clampedStack: string = clamp(
+      stackTrace,
+      MAX_STORED_STACK_TRACE_LENGTH,
+    );
+
+    let parsedFrames: string = "[]";
+    if (clampedStack) {
+      try {
+        const parsed: ParsedStackTrace = StackTraceParser.parse(clampedStack);
+        parsedFrames = JSON.stringify(parsed.frames);
+      } catch {
+        parsedFrames = "[]";
+      }
+    }
+
+    return {
+      message: clamp(message, MAX_MESSAGE_LENGTH),
+      exceptionType: exceptionType,
+      stackTrace: clampedStack,
+      parsedFrames: parsedFrames,
+      escaped: toNullableBoolean(attributes["exception.escaped"]),
+    };
+  }
+
+  private static extractFromBody(body: string): ExtractedLogException | null {
+    if (!body) {
+      return null;
+    }
+
+    const sliced: string =
+      body.length > MAX_PARSE_BODY_LENGTH
+        ? body.slice(0, MAX_PARSE_BODY_LENGTH)
+        : body;
+
+    if (!LOOKS_LIKE_STACK_TRACE.test(sliced)) {
+      return null;
+    }
+
+    const parsed: ParsedStackTrace = StackTraceParser.parse(sliced);
+
+    /*
+     * Require at least one parsed frame. A signature match with zero frames is
+     * usually prose that merely mentions "...Error:" / "...Exception" — not an
+     * actual stack trace. Path A is the path allowed to emit without frames.
+     */
+    if (!parsed.frames || parsed.frames.length === 0) {
+      return null;
+    }
+
+    const header: { exceptionType: string; message: string } =
+      LogExceptionExtractor.parseHeader(sliced);
+
+    return {
+      message: clamp(header.message, MAX_MESSAGE_LENGTH),
+      exceptionType: header.exceptionType,
+      stackTrace: clamp(sliced, MAX_STORED_STACK_TRACE_LENGTH),
+      parsedFrames: JSON.stringify(parsed.frames),
+      escaped: null,
+    };
+  }
+
+  /**
+   * Best-effort derivation of exceptionType + message from a raw stack-trace
+   * body. A stable, clean exceptionType improves grouping and exception-monitor
+   * targeting, but a miss is harmless — the fingerprint also uses the message
+   * and the normalized stack trace.
+   */
+  private static parseHeader(body: string): {
+    exceptionType: string;
+    message: string;
+  } {
+    const lines: Array<string> = body
+      .split("\n")
+      .map((l: string) => {
+        return l.trim();
+      })
+      .filter((l: string) => {
+        return l.length > 0;
+      });
+
+    if (lines.length === 0) {
+      return { exceptionType: "", message: "" };
+    }
+
+    /*
+     * Python: header is "Traceback (most recent call last):"; the "Type: message"
+     * line is at the BOTTOM of the traceback, so scan upward for it.
+     */
+    if (PYTHON_TRACEBACK_HEADER.test(lines[0]!)) {
+      for (let i: number = lines.length - 1; i >= 0; i--) {
+        const candidate: RegExpMatchArray | null = lines[i]!.match(TYPED_ERROR);
+        if (candidate) {
+          return {
+            exceptionType: candidate[1] || "",
+            message: (candidate[2] || "").trim(),
+          };
+        }
+      }
+      // Truncated traceback with no type line — keep the last line as message.
+      return { exceptionType: "", message: lines[lines.length - 1]! };
+    }
+
+    // Strip Java's "Exception in thread "name"" prefix if present.
+    let firstLine: string = lines[0]!;
+    const threadMatch: RegExpMatchArray | null =
+      firstLine.match(JAVA_THREAD_PREFIX);
+    if (threadMatch) {
+      firstLine = threadMatch[1]!.trim();
+    }
+
+    const goMatch: RegExpMatchArray | null = firstLine.match(GO_PANIC);
+    if (goMatch) {
+      return { exceptionType: "panic", message: (goMatch[1] || "").trim() };
+    }
+
+    // "TypeError: msg", "java.lang.NullPointerException: msg", "...Exception"
+    const typedMatch: RegExpMatchArray | null = firstLine.match(TYPED_ERROR);
+    if (typedMatch) {
+      return {
+        exceptionType: typedMatch[1] || "",
+        message: (typedMatch[2] || "").trim(),
+      };
+    }
+
+    // Generic qualified "pkg.Sub.Type: msg"
+    const qualifiedMatch: RegExpMatchArray | null =
+      firstLine.match(QUALIFIED_TYPE);
+    if (qualifiedMatch) {
+      return {
+        exceptionType: qualifiedMatch[1] || "",
+        message: (qualifiedMatch[2] || "").trim(),
+      };
+    }
+
+    return { exceptionType: "", message: firstLine };
+  }
+}
+
+function asString(value: JSONValue | undefined): string {
+  if (typeof value === "string") {
+    return value;
+  }
+  return "";
+}
+
+function toNullableBoolean(value: JSONValue | undefined): boolean | null {
+  if (typeof value === "boolean") {
+    return value;
+  }
+  if (typeof value === "string") {
+    const normalized: string = value.trim().toLowerCase();
+    if (normalized === "true") {
+      return true;
+    }
+    if (normalized === "false") {
+      return false;
+    }
+  }
+  return null;
+}
+
+function clamp(value: string, max: number): string {
+  if (value && value.length > max) {
+    return value.slice(0, max);
+  }
+  return value;
+}