@oneuptime/common 11.5.3 → 11.5.5

package/Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.ts

@@ -0,0 +1,317 @@
+import ObjectID from "../../../../../../Types/ObjectID";
+import OneUptimeDate from "../../../../../../Types/Date";
+import SortOrder from "../../../../../../Types/BaseDatabase/SortOrder";
+import InBetween from "../../../../../../Types/BaseDatabase/InBetween";
+import PositiveNumber from "../../../../../../Types/PositiveNumber";
+import { JSONObject } from "../../../../../../Types/JSON";
+import { LIMIT_PER_PROJECT } from "../../../../../../Types/Database/LimitMax";
+import AIInsightType from "../../../../../../Types/AI/AIInsightType";
+import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
+import TelemetryException from "../../../../../../Models/DatabaseModels/TelemetryException";
+import Service from "../../../../../../Models/DatabaseModels/Service";
+import TelemetryExceptionService from "../../../../../Services/TelemetryExceptionService";
+import ExceptionInstanceService from "../../../../../Services/ExceptionInstanceService";
+import ServiceService from "../../../../../Services/ServiceService";
+import QueryHelper from "../../../../../Types/Database/QueryHelper";
+import {
+  InsightCandidate,
+  InsightDetector,
+  InsightScanContext,
+} from "../Types";
+
+/*
+ * ExceptionSpike — an ESTABLISHED exception (first seen more than a day
+ * ago) whose recent hourly occurrence rate blew past its own 24-hour
+ * baseline, including the dormant-awakening case (an old exception that was
+ * quiet all day suddenly firing). Candidates come cheap from Postgres
+ * (group table); the actual rates come from two ClickHouse counts per
+ * candidate (fingerprint bloom index + time range — the exception-monitor
+ * count idiom). Deterministic — no LLM anywhere in this detector.
+ */
+
+// The spike window: occurrences in the last hour, compared to...
+export const EXCEPTION_SPIKE_RECENT_WINDOW_HOURS: number = 1;
+
+/*
+ * ...the exception's own average hourly rate over the 24 hours BEFORE the
+ * spike window. Self-baselining means a chronically noisy exception does
+ * not alert on its normal noise.
+ */
+export const EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS: number = 24;
+
+/*
+ * Absolute floor: below 10 occurrences/hour a "5x spike" can be 2→10 raw
+ * events — statistically meaningless and guaranteed inbox noise.
+ */
+export const EXCEPTION_SPIKE_MIN_RECENT_COUNT: number = 10;
+
+/*
+ * Rate multiplier that counts as a spike. The baseline hourly average is
+ * floored at 1 (see evaluateSpike) so near-zero baselines don't produce
+ * infinite multipliers; the dormant case (prior24 == 0, recent >= 10) then
+ * falls out of the same rule automatically.
+ */
+export const EXCEPTION_SPIKE_MIN_MULTIPLIER: number = 5;
+
+// A >= 10x spike is a step change, not a fluctuation — escalate to High.
+export const EXCEPTION_SPIKE_HIGH_SEVERITY_MULTIPLIER: number = 10;
+
+/*
+ * Candidate cap per scan tick: two ClickHouse counts per candidate, so 25
+ * candidates = 50 bounded point queries. Sorted by lifetime occurrence
+ * count so the perennial heavy hitters are always inspected first.
+ */
+export const EXCEPTION_SPIKE_CANDIDATE_LIMIT: number = 25;
+
+// Titles must stay scannable in a list — truncate long exception messages.
+export const EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH: number = 80;
+
+export interface ExceptionSpikeDecision {
+  isSpike: boolean;
+  // recent count / max(baseline hourly average, 1).
+  multiplier: number;
+  severity: AIInsightSeverity;
+  // prior-window count / EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS (unfloored).
+  baselineHourlyAverage: number;
+  // Zero occurrences in the whole baseline window, then a burst.
+  isDormantAwakening: boolean;
+}
+
+export default class ExceptionSpikeDetector implements InsightDetector {
+  public insightType: AIInsightType = AIInsightType.ExceptionSpike;
+
+  /*
+   * Pure decision: recent-hour count vs the prior 24h window count.
+   * Spike ⇔ recent >= EXCEPTION_SPIKE_MIN_RECENT_COUNT AND
+   * recent >= EXCEPTION_SPIKE_MIN_MULTIPLIER * max(prior/24, 1).
+   */
+  public static evaluateSpike(
+    recentCount: number,
+    priorWindowCount: number,
+  ): ExceptionSpikeDecision {
+    const baselineHourlyAverage: number =
+      priorWindowCount / EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS;
+    const effectiveBaseline: number = Math.max(baselineHourlyAverage, 1);
+    const multiplier: number = recentCount / effectiveBaseline;
+
+    const isSpike: boolean =
+      recentCount >= EXCEPTION_SPIKE_MIN_RECENT_COUNT &&
+      multiplier >= EXCEPTION_SPIKE_MIN_MULTIPLIER;
+
+    return {
+      isSpike,
+      multiplier,
+      severity:
+        multiplier >= EXCEPTION_SPIKE_HIGH_SEVERITY_MULTIPLIER
+          ? AIInsightSeverity.High
+          : AIInsightSeverity.Medium,
+      baselineHourlyAverage,
+      isDormantAwakening:
+        priorWindowCount === 0 &&
+        recentCount >= EXCEPTION_SPIKE_MIN_RECENT_COUNT,
+    };
+  }
+
+  /*
+   * Human label for the exception: the type when present, else the first
+   * EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH characters of the message.
+   */
+  public static buildExceptionLabel(
+    exceptionType: string | undefined,
+    message: string | undefined,
+  ): string {
+    if (exceptionType && exceptionType.trim().length > 0) {
+      return exceptionType.trim();
+    }
+    const trimmedMessage: string = (message || "").trim();
+    if (trimmedMessage.length === 0) {
+      return "Unknown exception";
+    }
+    if (trimmedMessage.length <= EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH) {
+      return trimmedMessage;
+    }
+    return `${trimmedMessage.slice(0, EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH)}…`;
+  }
+
+  // Stable dedupe key, keyed on the exception GROUP id (ingest dedupe unit).
+  public static buildFingerprint(telemetryExceptionId: string): string {
+    return `exception-spike:${telemetryExceptionId}`;
+  }
+
+  public async detect(
+    context: InsightScanContext,
+  ): Promise<Array<InsightCandidate>> {
+    const recentWindowStart: Date = OneUptimeDate.addRemoveHours(
+      context.now,
+      -1 * EXCEPTION_SPIKE_RECENT_WINDOW_HOURS,
+    );
+    const priorWindowStart: Date = OneUptimeDate.addRemoveHours(
+      recentWindowStart,
+      -1 * EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS,
+    );
+    const establishedBefore: Date = OneUptimeDate.addRemoveHours(
+      context.now,
+      -1 * EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS,
+    );
+
+    /*
+     * Candidates: established exception groups (born before the baseline
+     * window so they HAVE a baseline) that were active in the spike
+     * window. Anything younger is NewExceptionDetector's territory.
+     */
+    const candidateQuery: JSONObject = {
+      projectId: context.projectId,
+      lastSeenAt: QueryHelper.greaterThanEqualTo(recentWindowStart),
+      firstSeenAt: QueryHelper.lessThan(establishedBefore),
+      isResolved: false,
+      isArchived: false,
+    };
+
+    const exceptions: Array<TelemetryException> =
+      await TelemetryExceptionService.findBy({
+        query: candidateQuery as never,
+        select: {
+          _id: true,
+          message: true,
+          exceptionType: true,
+          fingerprint: true,
+          occuranceCount: true,
+          firstSeenAt: true,
+          primaryEntityId: true,
+        },
+        sort: {
+          occuranceCount: SortOrder.Descending,
+        },
+        limit: EXCEPTION_SPIKE_CANDIDATE_LIMIT,
+        skip: 0,
+        props: { isRoot: true },
+      });
+
+    const candidates: Array<InsightCandidate> = [];
+
+    for (const exception of exceptions) {
+      if (!exception.id || !exception.fingerprint) {
+        continue;
+      }
+
+      /*
+       * Two bounded ClickHouse counts (fingerprint bloom + time range —
+       * the MonitorTelemetryMonitor.monitorException idiom). Occurrence
+       * `time` is client-reported, so both windows use the same clock and
+       * skew cancels out within the comparison.
+       */
+      const recentCount: PositiveNumber =
+        await ExceptionInstanceService.countBy({
+          query: {
+            projectId: context.projectId,
+            fingerprint: exception.fingerprint,
+            time: new InBetween(recentWindowStart, context.now),
+          } as never,
+          skip: 0,
+          limit: LIMIT_PER_PROJECT,
+          props: { isRoot: true },
+        });
+
+      const priorCount: PositiveNumber = await ExceptionInstanceService.countBy(
+        {
+          query: {
+            projectId: context.projectId,
+            fingerprint: exception.fingerprint,
+            time: new InBetween(priorWindowStart, recentWindowStart),
+          } as never,
+          skip: 0,
+          limit: LIMIT_PER_PROJECT,
+          props: { isRoot: true },
+        },
+      );
+
+      const decision: ExceptionSpikeDecision =
+        ExceptionSpikeDetector.evaluateSpike(
+          recentCount.toNumber(),
+          priorCount.toNumber(),
+        );
+
+      if (!decision.isSpike) {
+        continue;
+      }
+
+      const serviceName: string | null = await this.resolveServiceName(
+        exception.primaryEntityId,
+      );
+
+      const label: string = ExceptionSpikeDetector.buildExceptionLabel(
+        exception.exceptionType,
+        exception.message,
+      );
+
+      const firstSeenAtIso: string | undefined = exception.firstSeenAt
+        ? exception.firstSeenAt.toISOString()
+        : undefined;
+
+      const detailLines: Array<string> = [
+        decision.isDormantAwakening
+          ? "**Dormant exception woke up**"
+          : "**Exception spike detected**",
+        "",
+        `- Exception: \`${label}\``,
+      ];
+      if (exception.message) {
+        detailLines.push(`- Message: ${exception.message}`);
+      }
+      if (serviceName) {
+        detailLines.push(`- Service: ${serviceName}`);
+      }
+      detailLines.push(
+        `- Occurrences in the last ${EXCEPTION_SPIKE_RECENT_WINDOW_HOURS}h: ${recentCount.toNumber()}`,
+        `- Baseline: ${decision.baselineHourlyAverage.toFixed(2)}/hour over the prior ${EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS}h (${priorCount.toNumber()} total)`,
+        `- Spike multiplier: ${decision.multiplier.toFixed(1)}x`,
+        `- Exception first seen: ${firstSeenAtIso || "unknown"} (lifetime occurrences: ${exception.occuranceCount || 0})`,
+      );
+
+      candidates.push({
+        insightType: AIInsightType.ExceptionSpike,
+        fingerprint: ExceptionSpikeDetector.buildFingerprint(
+          exception.id.toString(),
+        ),
+        title: `Exception spike: ${label} at ${decision.multiplier.toFixed(1)}x normal rate${serviceName ? ` in ${serviceName}` : ""}`,
+        detailMarkdown: detailLines.join("\n"),
+        severity: decision.severity,
+        serviceName: serviceName || undefined,
+        telemetryServiceId: serviceName ? exception.primaryEntityId : undefined,
+        telemetryExceptionId: exception.id,
+        evidence: {
+          exception: {
+            exceptionMessage: exception.message,
+            exceptionType: exception.exceptionType,
+            recentOccurrenceCount: recentCount.toNumber(),
+            baselineHourlyAverage: decision.baselineHourlyAverage,
+            spikeMultiplier: decision.multiplier,
+            totalOccurrenceCount: exception.occuranceCount,
+            firstSeenAt: firstSeenAtIso,
+          },
+        },
+      });
+    }
+
+    return candidates;
+  }
+
+  /*
+   * Best-effort service attribution: primaryEntityId is polymorphic and
+   * findOneById returns null for non-Service kinds — the insight then
+   * ships without a service name.
+   */
+  private async resolveServiceName(
+    primaryEntityId: ObjectID | undefined,
+  ): Promise<string | null> {
+    if (!primaryEntityId) {
+      return null;
+    }
+    const service: Service | null = await ServiceService.findOneById({
+      id: primaryEntityId,
+      select: { name: true },
+      props: { isRoot: true },
+    });
+    return service?.name || null;
+  }
+}

package/Server/Utils/AI/SRE/Insights/Detectors/Index.ts

@@ -0,0 +1,27 @@
+import { InsightDetector } from "../Types";
+import NewExceptionDetector from "./NewExceptionDetector";
+import ExceptionSpikeDetector from "./ExceptionSpikeDetector";
+import ErrorLogSpikeDetector from "./ErrorLogSpikeDetector";
+import TraceLatencyRegressionDetector from "./TraceLatencyRegressionDetector";
+import MetricDriftDetector from "./MetricDriftDetector";
+
+/*
+ * The detector registry. All five deterministic sensors — no LLM in any of
+ * them. Order matters for the scanner's per-project new-insight cap: the
+ * signals most likely to be actionable code problems (exceptions, then
+ * latency) come before the softer volume/drift signals, so when the cap
+ * bites it drops the softest findings first. The scanner runs every
+ * detector with per-detector error isolation — a detector that throws
+ * must not silence its siblings.
+ */
+export default class InsightDetectors {
+  public static getAllDetectors(): Array<InsightDetector> {
+    return [
+      new NewExceptionDetector(),
+      new ExceptionSpikeDetector(),
+      new TraceLatencyRegressionDetector(),
+      new ErrorLogSpikeDetector(),
+      new MetricDriftDetector(),
+    ];
+  }
+}

package/Server/Utils/AI/SRE/Insights/Detectors/MetricDriftDetector.ts

@@ -0,0 +1,221 @@
+import AIInsightType from "../../../../../../Types/AI/AIInsightType";
+import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
+import MetricBaselineService, {
+  MetricDriftWindowRow,
+  MetricBaselineService as MetricBaselineServiceClass,
+} from "../../../../../Services/MetricBaselineService";
+import {
+  InsightCandidate,
+  InsightDetector,
+  InsightScanContext,
+} from "../Types";
+
+/*
+ * MetricDrift — a metric's mean over the last 7 days drifted substantially
+ * from its mean over the prior 7 days, per (metric name, entity) cell. The
+ * data comes from two GROUP BY finalization queries over the
+ * MetricBaselineHourly materialized view (already maintained for anomaly
+ * monitors — this detector adds no write load); the recent/prior join and
+ * the drift decision are pure functions here so they are unit-testable
+ * without any ClickHouse.
+ *
+ * Drift severity is always Low and NEVER routes to an auto-fix: direction
+ * and goodness are unknowable without semantics (a 60% drop in latency is
+ * a win; a 60% drop in throughput is an outage). It is a "look here"
+ * signal, nothing more. Deterministic — no LLM anywhere in this detector.
+ */
+
+/*
+ * Both windows need this many raw samples per cell before their means are
+ * comparable — below it a single noisy hour can swing the mean by more
+ * than the drift threshold.
+ */
+export const METRIC_DRIFT_MIN_SAMPLES_PER_WINDOW: number = 100;
+
+/*
+ * Relative mean change (|recent-prior| / |prior|) that counts as drift.
+ * 50% is far outside normal week-to-week variation for the sample-count-
+ * gated cells, so the inbox only sees step changes.
+ */
+export const METRIC_DRIFT_MIN_RELATIVE_CHANGE: number = 0.5;
+
+/*
+ * At most this many drift insights per scan tick, ranked by |relative
+ * change|. A platform-wide shift moves hundreds of metrics at once — the
+ * top movers tell the story, the rest are echoes.
+ */
+export const METRIC_DRIFT_MAX_INSIGHTS: number = 5;
+
+/*
+ * Rows fetched per drift window (recent/prior), ranked by sample count.
+ * Bounded because the detector joins the windows in JS. Mirrors
+ * MetricBaselineService.DRIFT_MAX_ROWS_PER_WINDOW — the service clamps to
+ * that cap regardless.
+ */
+export const METRIC_DRIFT_ROWS_PER_WINDOW: number =
+  MetricBaselineServiceClass.DRIFT_MAX_ROWS_PER_WINDOW;
+
+// One joined-and-qualified drift finding, ready to become an insight.
+export interface MetricDriftFinding {
+  metricName: string;
+  primaryEntityId: string;
+  recentMean: number;
+  priorMean: number;
+  // Signed: (recent - prior) / |prior|. Positive = the mean went up.
+  relativeChange: number;
+  recentSampleCount: number;
+  priorSampleCount: number;
+}
+
+export default class MetricDriftDetector implements InsightDetector {
+  public insightType: AIInsightType = AIInsightType.MetricDrift;
+
+  /*
+   * Pure join + decision: pair each (name, entity) cell's recent and
+   * prior rows, gate on sample counts, and keep cells whose relative mean
+   * change clears the threshold. Cells with a zero prior mean are skipped
+   * — relative change against a zero baseline is undefined, and
+   * "metric appeared" is NewException/ErrorLogSpike territory, not drift.
+   * Returns the top METRIC_DRIFT_MAX_INSIGHTS movers by |relative change|.
+   */
+  public static evaluateDrift(
+    rows: Array<MetricDriftWindowRow>,
+  ): Array<MetricDriftFinding> {
+    const recentByCell: Map<string, MetricDriftWindowRow> = new Map();
+    const priorByCell: Map<string, MetricDriftWindowRow> = new Map();
+
+    for (const row of rows) {
+      // \u0000 cannot appear in either component — collision-free key.
+      const cellKey: string = `${row.name}\u0000${row.primaryEntityId}`;
+      if (row.window === "recent") {
+        recentByCell.set(cellKey, row);
+      } else {
+        priorByCell.set(cellKey, row);
+      }
+    }
+
+    const findings: Array<MetricDriftFinding> = [];
+
+    for (const [cellKey, recent] of recentByCell) {
+      const prior: MetricDriftWindowRow | undefined = priorByCell.get(cellKey);
+      if (!prior) {
+        continue;
+      }
+      if (
+        recent.sampleCount < METRIC_DRIFT_MIN_SAMPLES_PER_WINDOW ||
+        prior.sampleCount < METRIC_DRIFT_MIN_SAMPLES_PER_WINDOW
+      ) {
+        continue;
+      }
+      if (prior.mean === 0) {
+        continue;
+      }
+
+      const relativeChange: number =
+        (recent.mean - prior.mean) / Math.abs(prior.mean);
+
+      if (Math.abs(relativeChange) < METRIC_DRIFT_MIN_RELATIVE_CHANGE) {
+        continue;
+      }
+
+      findings.push({
+        metricName: recent.name,
+        primaryEntityId: recent.primaryEntityId,
+        recentMean: recent.mean,
+        priorMean: prior.mean,
+        relativeChange,
+        recentSampleCount: recent.sampleCount,
+        priorSampleCount: prior.sampleCount,
+      });
+    }
+
+    findings.sort((a: MetricDriftFinding, b: MetricDriftFinding): number => {
+      if (Math.abs(b.relativeChange) !== Math.abs(a.relativeChange)) {
+        return Math.abs(b.relativeChange) - Math.abs(a.relativeChange);
+      }
+      // Deterministic tie-break so scan ticks are reproducible.
+      const aKey: string = `${a.metricName}\u0000${a.primaryEntityId}`;
+      const bKey: string = `${b.metricName}\u0000${b.primaryEntityId}`;
+      return aKey.localeCompare(bKey);
+    });
+
+    return findings.slice(0, METRIC_DRIFT_MAX_INSIGHTS);
+  }
+
+  // Signed percent label, e.g. "+62%" / "-71%".
+  public static formatChangePercent(relativeChange: number): string {
+    const percent: number = Math.round(relativeChange * 100);
+    return `${percent >= 0 ? "+" : ""}${percent}%`;
+  }
+
+  // Stable dedupe key per (metric name, entity) cell.
+  public static buildFingerprint(
+    metricName: string,
+    primaryEntityId: string,
+  ): string {
+    return `metric-drift:${metricName}:${primaryEntityId}`;
+  }
+
+  public async detect(
+    context: InsightScanContext,
+  ): Promise<Array<InsightCandidate>> {
+    const rows: Array<MetricDriftWindowRow> =
+      await MetricBaselineService.getWeekOverWeekDrift({
+        projectId: context.projectId,
+        limitPerWindow: METRIC_DRIFT_ROWS_PER_WINDOW,
+      });
+
+    const findings: Array<MetricDriftFinding> =
+      MetricDriftDetector.evaluateDrift(rows);
+
+    return findings.map((finding: MetricDriftFinding): InsightCandidate => {
+      const changeLabel: string = MetricDriftDetector.formatChangePercent(
+        finding.relativeChange,
+      );
+
+      const detailLines: Array<string> = [
+        "**Metric drift detected (week-over-week)**",
+        "",
+        `- Metric: \`${finding.metricName}\``,
+      ];
+      if (finding.primaryEntityId) {
+        detailLines.push(`- Entity: \`${finding.primaryEntityId}\``);
+      }
+      detailLines.push(
+        `- Recent 7-day mean: ${finding.recentMean.toFixed(4)} (${finding.recentSampleCount} samples)`,
+        `- Prior 7-day mean: ${finding.priorMean.toFixed(4)} (${finding.priorSampleCount} samples)`,
+        `- Relative change: ${changeLabel}`,
+        "",
+        "Direction and goodness are metric-specific — this is a heads-up, not an alarm.",
+      );
+
+      return {
+        insightType: AIInsightType.MetricDrift,
+        fingerprint: MetricDriftDetector.buildFingerprint(
+          finding.metricName,
+          finding.primaryEntityId,
+        ),
+        title: `Metric drift: ${finding.metricName} ${changeLabel} week-over-week`,
+        detailMarkdown: detailLines.join("\n"),
+        /*
+         * Always Low: without metric semantics the detector cannot know
+         * whether the drift is good or bad — it must never outrank a
+         * concrete exception or latency finding.
+         */
+        severity: AIInsightSeverity.Low,
+        metricName: finding.metricName,
+        evidence: {
+          metricDrift: {
+            metricName: finding.metricName,
+            primaryEntityId: finding.primaryEntityId || undefined,
+            recentWeekMean: finding.recentMean,
+            priorWeekMean: finding.priorMean,
+            relativeChangePercent: finding.relativeChange * 100,
+            recentSampleCount: finding.recentSampleCount,
+            priorSampleCount: finding.priorSampleCount,
+          },
+        },
+      };
+    });
+  }
+}