@oneuptime/common 11.5.3 → 11.5.5

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.js

@@ -0,0 +1,226 @@
+import OneUptimeDate from "../../../../../../Types/Date";
+import SortOrder from "../../../../../../Types/BaseDatabase/SortOrder";
+import InBetween from "../../../../../../Types/BaseDatabase/InBetween";
+import { LIMIT_PER_PROJECT } from "../../../../../../Types/Database/LimitMax";
+import AIInsightType from "../../../../../../Types/AI/AIInsightType";
+import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
+import TelemetryExceptionService from "../../../../../Services/TelemetryExceptionService";
+import ExceptionInstanceService from "../../../../../Services/ExceptionInstanceService";
+import ServiceService from "../../../../../Services/ServiceService";
+import QueryHelper from "../../../../../Types/Database/QueryHelper";
+/*
+ * ExceptionSpike — an ESTABLISHED exception (first seen more than a day
+ * ago) whose recent hourly occurrence rate blew past its own 24-hour
+ * baseline, including the dormant-awakening case (an old exception that was
+ * quiet all day suddenly firing). Candidates come cheap from Postgres
+ * (group table); the actual rates come from two ClickHouse counts per
+ * candidate (fingerprint bloom index + time range — the exception-monitor
+ * count idiom). Deterministic — no LLM anywhere in this detector.
+ */
+// The spike window: occurrences in the last hour, compared to...
+export const EXCEPTION_SPIKE_RECENT_WINDOW_HOURS = 1;
+/*
+ * ...the exception's own average hourly rate over the 24 hours BEFORE the
+ * spike window. Self-baselining means a chronically noisy exception does
+ * not alert on its normal noise.
+ */
+export const EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS = 24;
+/*
+ * Absolute floor: below 10 occurrences/hour a "5x spike" can be 2→10 raw
+ * events — statistically meaningless and guaranteed inbox noise.
+ */
+export const EXCEPTION_SPIKE_MIN_RECENT_COUNT = 10;
+/*
+ * Rate multiplier that counts as a spike. The baseline hourly average is
+ * floored at 1 (see evaluateSpike) so near-zero baselines don't produce
+ * infinite multipliers; the dormant case (prior24 == 0, recent >= 10) then
+ * falls out of the same rule automatically.
+ */
+export const EXCEPTION_SPIKE_MIN_MULTIPLIER = 5;
+// A >= 10x spike is a step change, not a fluctuation — escalate to High.
+export const EXCEPTION_SPIKE_HIGH_SEVERITY_MULTIPLIER = 10;
+/*
+ * Candidate cap per scan tick: two ClickHouse counts per candidate, so 25
+ * candidates = 50 bounded point queries. Sorted by lifetime occurrence
+ * count so the perennial heavy hitters are always inspected first.
+ */
+export const EXCEPTION_SPIKE_CANDIDATE_LIMIT = 25;
+// Titles must stay scannable in a list — truncate long exception messages.
+export const EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH = 80;
+export default class ExceptionSpikeDetector {
+    constructor() {
+        this.insightType = AIInsightType.ExceptionSpike;
+    }
+    /*
+     * Pure decision: recent-hour count vs the prior 24h window count.
+     * Spike ⇔ recent >= EXCEPTION_SPIKE_MIN_RECENT_COUNT AND
+     * recent >= EXCEPTION_SPIKE_MIN_MULTIPLIER * max(prior/24, 1).
+     */
+    static evaluateSpike(recentCount, priorWindowCount) {
+        const baselineHourlyAverage = priorWindowCount / EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS;
+        const effectiveBaseline = Math.max(baselineHourlyAverage, 1);
+        const multiplier = recentCount / effectiveBaseline;
+        const isSpike = recentCount >= EXCEPTION_SPIKE_MIN_RECENT_COUNT &&
+            multiplier >= EXCEPTION_SPIKE_MIN_MULTIPLIER;
+        return {
+            isSpike,
+            multiplier,
+            severity: multiplier >= EXCEPTION_SPIKE_HIGH_SEVERITY_MULTIPLIER
+                ? AIInsightSeverity.High
+                : AIInsightSeverity.Medium,
+            baselineHourlyAverage,
+            isDormantAwakening: priorWindowCount === 0 &&
+                recentCount >= EXCEPTION_SPIKE_MIN_RECENT_COUNT,
+        };
+    }
+    /*
+     * Human label for the exception: the type when present, else the first
+     * EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH characters of the message.
+     */
+    static buildExceptionLabel(exceptionType, message) {
+        if (exceptionType && exceptionType.trim().length > 0) {
+            return exceptionType.trim();
+        }
+        const trimmedMessage = (message || "").trim();
+        if (trimmedMessage.length === 0) {
+            return "Unknown exception";
+        }
+        if (trimmedMessage.length <= EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH) {
+            return trimmedMessage;
+        }
+        return `${trimmedMessage.slice(0, EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH)}…`;
+    }
+    // Stable dedupe key, keyed on the exception GROUP id (ingest dedupe unit).
+    static buildFingerprint(telemetryExceptionId) {
+        return `exception-spike:${telemetryExceptionId}`;
+    }
+    async detect(context) {
+        const recentWindowStart = OneUptimeDate.addRemoveHours(context.now, -1 * EXCEPTION_SPIKE_RECENT_WINDOW_HOURS);
+        const priorWindowStart = OneUptimeDate.addRemoveHours(recentWindowStart, -1 * EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS);
+        const establishedBefore = OneUptimeDate.addRemoveHours(context.now, -1 * EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS);
+        /*
+         * Candidates: established exception groups (born before the baseline
+         * window so they HAVE a baseline) that were active in the spike
+         * window. Anything younger is NewExceptionDetector's territory.
+         */
+        const candidateQuery = {
+            projectId: context.projectId,
+            lastSeenAt: QueryHelper.greaterThanEqualTo(recentWindowStart),
+            firstSeenAt: QueryHelper.lessThan(establishedBefore),
+            isResolved: false,
+            isArchived: false,
+        };
+        const exceptions = await TelemetryExceptionService.findBy({
+            query: candidateQuery,
+            select: {
+                _id: true,
+                message: true,
+                exceptionType: true,
+                fingerprint: true,
+                occuranceCount: true,
+                firstSeenAt: true,
+                primaryEntityId: true,
+            },
+            sort: {
+                occuranceCount: SortOrder.Descending,
+            },
+            limit: EXCEPTION_SPIKE_CANDIDATE_LIMIT,
+            skip: 0,
+            props: { isRoot: true },
+        });
+        const candidates = [];
+        for (const exception of exceptions) {
+            if (!exception.id || !exception.fingerprint) {
+                continue;
+            }
+            /*
+             * Two bounded ClickHouse counts (fingerprint bloom + time range —
+             * the MonitorTelemetryMonitor.monitorException idiom). Occurrence
+             * `time` is client-reported, so both windows use the same clock and
+             * skew cancels out within the comparison.
+             */
+            const recentCount = await ExceptionInstanceService.countBy({
+                query: {
+                    projectId: context.projectId,
+                    fingerprint: exception.fingerprint,
+                    time: new InBetween(recentWindowStart, context.now),
+                },
+                skip: 0,
+                limit: LIMIT_PER_PROJECT,
+                props: { isRoot: true },
+            });
+            const priorCount = await ExceptionInstanceService.countBy({
+                query: {
+                    projectId: context.projectId,
+                    fingerprint: exception.fingerprint,
+                    time: new InBetween(priorWindowStart, recentWindowStart),
+                },
+                skip: 0,
+                limit: LIMIT_PER_PROJECT,
+                props: { isRoot: true },
+            });
+            const decision = ExceptionSpikeDetector.evaluateSpike(recentCount.toNumber(), priorCount.toNumber());
+            if (!decision.isSpike) {
+                continue;
+            }
+            const serviceName = await this.resolveServiceName(exception.primaryEntityId);
+            const label = ExceptionSpikeDetector.buildExceptionLabel(exception.exceptionType, exception.message);
+            const firstSeenAtIso = exception.firstSeenAt
+                ? exception.firstSeenAt.toISOString()
+                : undefined;
+            const detailLines = [
+                decision.isDormantAwakening
+                    ? "**Dormant exception woke up**"
+                    : "**Exception spike detected**",
+                "",
+                `- Exception: \`${label}\``,
+            ];
+            if (exception.message) {
+                detailLines.push(`- Message: ${exception.message}`);
+            }
+            if (serviceName) {
+                detailLines.push(`- Service: ${serviceName}`);
+            }
+            detailLines.push(`- Occurrences in the last ${EXCEPTION_SPIKE_RECENT_WINDOW_HOURS}h: ${recentCount.toNumber()}`, `- Baseline: ${decision.baselineHourlyAverage.toFixed(2)}/hour over the prior ${EXCEPTION_SPIKE_BASELINE_WINDOW_HOURS}h (${priorCount.toNumber()} total)`, `- Spike multiplier: ${decision.multiplier.toFixed(1)}x`, `- Exception first seen: ${firstSeenAtIso || "unknown"} (lifetime occurrences: ${exception.occuranceCount || 0})`);
+            candidates.push({
+                insightType: AIInsightType.ExceptionSpike,
+                fingerprint: ExceptionSpikeDetector.buildFingerprint(exception.id.toString()),
+                title: `Exception spike: ${label} at ${decision.multiplier.toFixed(1)}x normal rate${serviceName ? ` in ${serviceName}` : ""}`,
+                detailMarkdown: detailLines.join("\n"),
+                severity: decision.severity,
+                serviceName: serviceName || undefined,
+                telemetryServiceId: serviceName ? exception.primaryEntityId : undefined,
+                telemetryExceptionId: exception.id,
+                evidence: {
+                    exception: {
+                        exceptionMessage: exception.message,
+                        exceptionType: exception.exceptionType,
+                        recentOccurrenceCount: recentCount.toNumber(),
+                        baselineHourlyAverage: decision.baselineHourlyAverage,
+                        spikeMultiplier: decision.multiplier,
+                        totalOccurrenceCount: exception.occuranceCount,
+                        firstSeenAt: firstSeenAtIso,
+                    },
+                },
+            });
+        }
+        return candidates;
+    }
+    /*
+     * Best-effort service attribution: primaryEntityId is polymorphic and
+     * findOneById returns null for non-Service kinds — the insight then
+     * ships without a service name.
+     */
+    async resolveServiceName(primaryEntityId) {
+        if (!primaryEntityId) {
+            return null;
+        }
+        const service = await ServiceService.findOneById({
+            id: primaryEntityId,
+            select: { name: true },
+            props: { isRoot: true },
+        });
+        return (service === null || service === void 0 ? void 0 : service.name) || null;
+    }
+}
+//# sourceMappingURL=ExceptionSpikeDetector.js.map

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ExceptionSpikeDetector.js","sourceRoot":"","sources":["../../../../../../../../Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.ts"],"names":[],"mappings":"AACA,OAAO,aAAa,MAAM,8BAA8B,CAAC;AACzD,OAAO,SAAS,MAAM,gDAAgD,CAAC;AACvE,OAAO,SAAS,MAAM,gDAAgD,CAAC;AAGvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2CAA2C,CAAC;AAC9E,OAAO,aAAa,MAAM,0CAA0C,CAAC;AACrE,OAAO,iBAAiB,MAAM,8CAA8C,CAAC;AAG7E,OAAO,yBAAyB,MAAM,mDAAmD,CAAC;AAC1F,OAAO,wBAAwB,MAAM,kDAAkD,CAAC;AACxF,OAAO,cAAc,MAAM,wCAAwC,CAAC;AACpE,OAAO,WAAW,MAAM,2CAA2C,CAAC;AAOpE;;;;;;;;GAQG;AAEH,iEAAiE;AACjE,MAAM,CAAC,MAAM,mCAAmC,GAAW,CAAC,CAAC;AAE7D;;;;GAIG;AACH,MAAM,CAAC,MAAM,qCAAqC,GAAW,EAAE,CAAC;AAEhE;;;GAGG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAW,EAAE,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAW,CAAC,CAAC;AAExD,yEAAyE;AACzE,MAAM,CAAC,MAAM,wCAAwC,GAAW,EAAE,CAAC;AAEnE;;;;GAIG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAW,EAAE,CAAC;AAE1D,2EAA2E;AAC3E,MAAM,CAAC,MAAM,wCAAwC,GAAW,EAAE,CAAC;AAanE,MAAM,CAAC,OAAO,OAAO,sBAAsB;IAA3C;QACS,gBAAW,GAAkB,aAAa,CAAC,cAAc,CAAC;IA4OnE,CAAC;IA1OC;;;;OAIG;IACI,MAAM,CAAC,aAAa,CACzB,WAAmB,EACnB,gBAAwB;QAExB,MAAM,qBAAqB,GACzB,gBAAgB,GAAG,qCAAqC,CAAC;QAC3D,MAAM,iBAAiB,GAAW,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;QACrE,MAAM,UAAU,GAAW,WAAW,GAAG,iBAAiB,CAAC;QAE3D,MAAM,OAAO,GACX,WAAW,IAAI,gCAAgC;YAC/C,UAAU,IAAI,8BAA8B,CAAC;QAE/C,OAAO;YACL,OAAO;YACP,UAAU;YACV,QAAQ,EACN,UAAU,IAAI,wCAAwC;gBACpD,CAAC,CAAC,iBAAiB,CAAC,IAAI;gBACxB,CAAC,CAAC,iBAAiB,CAAC,MAAM;YAC9B,qBAAqB;YACrB,kBAAkB,EAChB,gBAAgB,KAAK,CAAC;gBACtB,WAAW,IAAI,gCAAgC;SAClD,CAAC;IACJ,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,mBAAmB,CAC/B,aAAiC,EACjC,OAA2B;QAE3B,IAAI,aAAa,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,OAAO,aAAa,CAAC,IAAI,EAAE,CAAC;QAC9B,CAAC;QACD,MAAM,cAAc,GAAW,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,mBAAmB,CAAC;QAC7B,CAAC;QACD,IAAI,cAAc,CAAC,MAAM,IAAI,wCAAwC,EAAE,CAAC;YACtE,OAAO,cAAc,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,wCAAwC,CAAC,GAAG,CAAC;IACjF,CAAC;IAED,2EAA2E;IACpE,MAAM,CAAC,gBAAgB,CAAC,oBAA4B;QACzD,OAAO,mBAAmB,oBAAoB,EAAE,CAAC;IACnD,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,OAA2B;QAE3B,MAAM,iBAAiB,GAAS,aAAa,CAAC,cAAc,CAC1D,OAAO,CAAC,GAAG,EACX,CAAC,CAAC,GAAG,mCAAmC,CACzC,CAAC;QACF,MAAM,gBAAgB,GAAS,aAAa,CAAC,cAAc,CACzD,iBAAiB,EACjB,CAAC,CAAC,GAAG,qCAAqC,CAC3C,CAAC;QACF,MAAM,iBAAiB,GAAS,aAAa,CAAC,cAAc,CAC1D,OAAO,CAAC,GAAG,EACX,CAAC,CAAC,GAAG,qCAAqC,CAC3C,CAAC;QAEF;;;;WAIG;QACH,MAAM,cAAc,GAAe;YACjC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,UAAU,EAAE,WAAW,CAAC,kBAAkB,CAAC,iBAAiB,CAAC;YAC7D,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC;YACpD,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,KAAK;SAClB,CAAC;QAEF,MAAM,UAAU,GACd,MAAM,yBAAyB,CAAC,MAAM,CAAC;YACrC,KAAK,EAAE,cAAuB;YAC9B,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI;gBACT,OAAO,EAAE,IAAI;gBACb,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,IAAI;gBACjB,cAAc,EAAE,IAAI;gBACpB,WAAW,EAAE,IAAI;gBACjB,eAAe,EAAE,IAAI;aACtB;YACD,IAAI,EAAE;gBACJ,cAAc,EAAE,SAAS,CAAC,UAAU;aACrC;YACD,KAAK,EAAE,+BAA+B;YACtC,IAAI,EAAE,CAAC;YACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QAEL,MAAM,UAAU,GAA4B,EAAE,CAAC;QAE/C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;gBAC5C,SAAS;YACX,CAAC;YAED;;;;;eAKG;YACH,MAAM,WAAW,GACf,MAAM,wBAAwB,CAAC,OAAO,CAAC;gBACrC,KAAK,EAAE;oBACL,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,IAAI,EAAE,IAAI,SAAS,CAAC,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC;iBAC3C;gBACV,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,iBAAiB;gBACxB,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;YAEL,MAAM,UAAU,GAAmB,MAAM,wBAAwB,CAAC,OAAO,CACvE;gBACE,KAAK,EAAE;oBACL,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,IAAI,EAAE,IAAI,SAAS,CAAC,gBAAgB,EAAE,iBAAiB,CAAC;iBAChD;gBACV,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,iBAAiB;gBACxB,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CACF,CAAC;YAEF,MAAM,QAAQ,GACZ,sBAAsB,CAAC,aAAa,CAClC,WAAW,CAAC,QAAQ,EAAE,EACtB,UAAU,CAAC,QAAQ,EAAE,CACtB,CAAC;YAEJ,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,MAAM,WAAW,GAAkB,MAAM,IAAI,CAAC,kBAAkB,CAC9D,SAAS,CAAC,eAAe,CAC1B,CAAC;YAEF,MAAM,KAAK,GAAW,sBAAsB,CAAC,mBAAmB,CAC9D,SAAS,CAAC,aAAa,EACvB,SAAS,CAAC,OAAO,CAClB,CAAC;YAEF,MAAM,cAAc,GAAuB,SAAS,CAAC,WAAW;gBAC9D,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,WAAW,EAAE;gBACrC,CAAC,CAAC,SAAS,CAAC;YAEd,MAAM,WAAW,GAAkB;gBACjC,QAAQ,CAAC,kBAAkB;oBACzB,CAAC,CAAC,+BAA+B;oBACjC,CAAC,CAAC,8BAA8B;gBAClC,EAAE;gBACF,kBAAkB,KAAK,IAAI;aAC5B,CAAC;YACF,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;gBACtB,WAAW,CAAC,IAAI,CAAC,cAAc,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,WAAW,EAAE,CAAC;gBAChB,WAAW,CAAC,IAAI,CAAC,cAAc,WAAW,EAAE,CAAC,CAAC;YAChD,CAAC;YACD,WAAW,CAAC,IAAI,CACd,6BAA6B,mCAAmC,MAAM,WAAW,CAAC,QAAQ,EAAE,EAAE,EAC9F,eAAe,QAAQ,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,qCAAqC,MAAM,UAAU,CAAC,QAAQ,EAAE,SAAS,EACzJ,uBAAuB,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EACxD,2BAA2B,cAAc,IAAI,SAAS,2BAA2B,SAAS,CAAC,cAAc,IAAI,CAAC,GAAG,CAClH,CAAC;YAEF,UAAU,CAAC,IAAI,CAAC;gBACd,WAAW,EAAE,aAAa,CAAC,cAAc;gBACzC,WAAW,EAAE,sBAAsB,CAAC,gBAAgB,CAClD,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,CACxB;gBACD,KAAK,EAAE,oBAAoB,KAAK,OAAO,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,WAAW,CAAC,CAAC,CAAC,OAAO,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC9H,cAAc,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;gBACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,WAAW,EAAE,WAAW,IAAI,SAAS;gBACrC,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;gBACvE,oBAAoB,EAAE,SAAS,CAAC,EAAE;gBAClC,QAAQ,EAAE;oBACR,SAAS,EAAE;wBACT,gBAAgB,EAAE,SAAS,CAAC,OAAO;wBACnC,aAAa,EAAE,SAAS,CAAC,aAAa;wBACtC,qBAAqB,EAAE,WAAW,CAAC,QAAQ,EAAE;wBAC7C,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;wBACrD,eAAe,EAAE,QAAQ,CAAC,UAAU;wBACpC,oBAAoB,EAAE,SAAS,CAAC,cAAc;wBAC9C,WAAW,EAAE,cAAc;qBAC5B;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,eAAqC;QAErC,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAAmB,MAAM,cAAc,CAAC,WAAW,CAAC;YAC/D,EAAE,EAAE,eAAe;YACnB,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;YACtB,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QACH,OAAO,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,KAAI,IAAI,CAAC;IAC/B,CAAC;CACF"}

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/Index.js

@@ -0,0 +1,26 @@
+import NewExceptionDetector from "./NewExceptionDetector";
+import ExceptionSpikeDetector from "./ExceptionSpikeDetector";
+import ErrorLogSpikeDetector from "./ErrorLogSpikeDetector";
+import TraceLatencyRegressionDetector from "./TraceLatencyRegressionDetector";
+import MetricDriftDetector from "./MetricDriftDetector";
+/*
+ * The detector registry. All five deterministic sensors — no LLM in any of
+ * them. Order matters for the scanner's per-project new-insight cap: the
+ * signals most likely to be actionable code problems (exceptions, then
+ * latency) come before the softer volume/drift signals, so when the cap
+ * bites it drops the softest findings first. The scanner runs every
+ * detector with per-detector error isolation — a detector that throws
+ * must not silence its siblings.
+ */
+export default class InsightDetectors {
+    static getAllDetectors() {
+        return [
+            new NewExceptionDetector(),
+            new ExceptionSpikeDetector(),
+            new TraceLatencyRegressionDetector(),
+            new ErrorLogSpikeDetector(),
+            new MetricDriftDetector(),
+        ];
+    }
+}
+//# sourceMappingURL=Index.js.map

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/Index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Index.js","sourceRoot":"","sources":["../../../../../../../../Server/Utils/AI/SRE/Insights/Detectors/Index.ts"],"names":[],"mappings":"AACA,OAAO,oBAAoB,MAAM,wBAAwB,CAAC;AAC1D,OAAO,sBAAsB,MAAM,0BAA0B,CAAC;AAC9D,OAAO,qBAAqB,MAAM,yBAAyB,CAAC;AAC5D,OAAO,8BAA8B,MAAM,kCAAkC,CAAC;AAC9E,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AAExD;;;;;;;;GAQG;AACH,MAAM,CAAC,OAAO,OAAO,gBAAgB;IAC5B,MAAM,CAAC,eAAe;QAC3B,OAAO;YACL,IAAI,oBAAoB,EAAE;YAC1B,IAAI,sBAAsB,EAAE;YAC5B,IAAI,8BAA8B,EAAE;YACpC,IAAI,qBAAqB,EAAE;YAC3B,IAAI,mBAAmB,EAAE;SAC1B,CAAC;IACJ,CAAC;CACF"}

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/MetricDriftDetector.js

@@ -0,0 +1,159 @@
+import AIInsightType from "../../../../../../Types/AI/AIInsightType";
+import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
+import MetricBaselineService, { MetricBaselineService as MetricBaselineServiceClass, } from "../../../../../Services/MetricBaselineService";
+/*
+ * MetricDrift — a metric's mean over the last 7 days drifted substantially
+ * from its mean over the prior 7 days, per (metric name, entity) cell. The
+ * data comes from two GROUP BY finalization queries over the
+ * MetricBaselineHourly materialized view (already maintained for anomaly
+ * monitors — this detector adds no write load); the recent/prior join and
+ * the drift decision are pure functions here so they are unit-testable
+ * without any ClickHouse.
+ *
+ * Drift severity is always Low and NEVER routes to an auto-fix: direction
+ * and goodness are unknowable without semantics (a 60% drop in latency is
+ * a win; a 60% drop in throughput is an outage). It is a "look here"
+ * signal, nothing more. Deterministic — no LLM anywhere in this detector.
+ */
+/*
+ * Both windows need this many raw samples per cell before their means are
+ * comparable — below it a single noisy hour can swing the mean by more
+ * than the drift threshold.
+ */
+export const METRIC_DRIFT_MIN_SAMPLES_PER_WINDOW = 100;
+/*
+ * Relative mean change (|recent-prior| / |prior|) that counts as drift.
+ * 50% is far outside normal week-to-week variation for the sample-count-
+ * gated cells, so the inbox only sees step changes.
+ */
+export const METRIC_DRIFT_MIN_RELATIVE_CHANGE = 0.5;
+/*
+ * At most this many drift insights per scan tick, ranked by |relative
+ * change|. A platform-wide shift moves hundreds of metrics at once — the
+ * top movers tell the story, the rest are echoes.
+ */
+export const METRIC_DRIFT_MAX_INSIGHTS = 5;
+/*
+ * Rows fetched per drift window (recent/prior), ranked by sample count.
+ * Bounded because the detector joins the windows in JS. Mirrors
+ * MetricBaselineService.DRIFT_MAX_ROWS_PER_WINDOW — the service clamps to
+ * that cap regardless.
+ */
+export const METRIC_DRIFT_ROWS_PER_WINDOW = MetricBaselineServiceClass.DRIFT_MAX_ROWS_PER_WINDOW;
+export default class MetricDriftDetector {
+    constructor() {
+        this.insightType = AIInsightType.MetricDrift;
+    }
+    /*
+     * Pure join + decision: pair each (name, entity) cell's recent and
+     * prior rows, gate on sample counts, and keep cells whose relative mean
+     * change clears the threshold. Cells with a zero prior mean are skipped
+     * — relative change against a zero baseline is undefined, and
+     * "metric appeared" is NewException/ErrorLogSpike territory, not drift.
+     * Returns the top METRIC_DRIFT_MAX_INSIGHTS movers by |relative change|.
+     */
+    static evaluateDrift(rows) {
+        const recentByCell = new Map();
+        const priorByCell = new Map();
+        for (const row of rows) {
+            // \u0000 cannot appear in either component — collision-free key.
+            const cellKey = `${row.name}\u0000${row.primaryEntityId}`;
+            if (row.window === "recent") {
+                recentByCell.set(cellKey, row);
+            }
+            else {
+                priorByCell.set(cellKey, row);
+            }
+        }
+        const findings = [];
+        for (const [cellKey, recent] of recentByCell) {
+            const prior = priorByCell.get(cellKey);
+            if (!prior) {
+                continue;
+            }
+            if (recent.sampleCount < METRIC_DRIFT_MIN_SAMPLES_PER_WINDOW ||
+                prior.sampleCount < METRIC_DRIFT_MIN_SAMPLES_PER_WINDOW) {
+                continue;
+            }
+            if (prior.mean === 0) {
+                continue;
+            }
+            const relativeChange = (recent.mean - prior.mean) / Math.abs(prior.mean);
+            if (Math.abs(relativeChange) < METRIC_DRIFT_MIN_RELATIVE_CHANGE) {
+                continue;
+            }
+            findings.push({
+                metricName: recent.name,
+                primaryEntityId: recent.primaryEntityId,
+                recentMean: recent.mean,
+                priorMean: prior.mean,
+                relativeChange,
+                recentSampleCount: recent.sampleCount,
+                priorSampleCount: prior.sampleCount,
+            });
+        }
+        findings.sort((a, b) => {
+            if (Math.abs(b.relativeChange) !== Math.abs(a.relativeChange)) {
+                return Math.abs(b.relativeChange) - Math.abs(a.relativeChange);
+            }
+            // Deterministic tie-break so scan ticks are reproducible.
+            const aKey = `${a.metricName}\u0000${a.primaryEntityId}`;
+            const bKey = `${b.metricName}\u0000${b.primaryEntityId}`;
+            return aKey.localeCompare(bKey);
+        });
+        return findings.slice(0, METRIC_DRIFT_MAX_INSIGHTS);
+    }
+    // Signed percent label, e.g. "+62%" / "-71%".
+    static formatChangePercent(relativeChange) {
+        const percent = Math.round(relativeChange * 100);
+        return `${percent >= 0 ? "+" : ""}${percent}%`;
+    }
+    // Stable dedupe key per (metric name, entity) cell.
+    static buildFingerprint(metricName, primaryEntityId) {
+        return `metric-drift:${metricName}:${primaryEntityId}`;
+    }
+    async detect(context) {
+        const rows = await MetricBaselineService.getWeekOverWeekDrift({
+            projectId: context.projectId,
+            limitPerWindow: METRIC_DRIFT_ROWS_PER_WINDOW,
+        });
+        const findings = MetricDriftDetector.evaluateDrift(rows);
+        return findings.map((finding) => {
+            const changeLabel = MetricDriftDetector.formatChangePercent(finding.relativeChange);
+            const detailLines = [
+                "**Metric drift detected (week-over-week)**",
+                "",
+                `- Metric: \`${finding.metricName}\``,
+            ];
+            if (finding.primaryEntityId) {
+                detailLines.push(`- Entity: \`${finding.primaryEntityId}\``);
+            }
+            detailLines.push(`- Recent 7-day mean: ${finding.recentMean.toFixed(4)} (${finding.recentSampleCount} samples)`, `- Prior 7-day mean: ${finding.priorMean.toFixed(4)} (${finding.priorSampleCount} samples)`, `- Relative change: ${changeLabel}`, "", "Direction and goodness are metric-specific — this is a heads-up, not an alarm.");
+            return {
+                insightType: AIInsightType.MetricDrift,
+                fingerprint: MetricDriftDetector.buildFingerprint(finding.metricName, finding.primaryEntityId),
+                title: `Metric drift: ${finding.metricName} ${changeLabel} week-over-week`,
+                detailMarkdown: detailLines.join("\n"),
+                /*
+                 * Always Low: without metric semantics the detector cannot know
+                 * whether the drift is good or bad — it must never outrank a
+                 * concrete exception or latency finding.
+                 */
+                severity: AIInsightSeverity.Low,
+                metricName: finding.metricName,
+                evidence: {
+                    metricDrift: {
+                        metricName: finding.metricName,
+                        primaryEntityId: finding.primaryEntityId || undefined,
+                        recentWeekMean: finding.recentMean,
+                        priorWeekMean: finding.priorMean,
+                        relativeChangePercent: finding.relativeChange * 100,
+                        recentSampleCount: finding.recentSampleCount,
+                        priorSampleCount: finding.priorSampleCount,
+                    },
+                },
+            };
+        });
+    }
+}
+//# sourceMappingURL=MetricDriftDetector.js.map

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/MetricDriftDetector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MetricDriftDetector.js","sourceRoot":"","sources":["../../../../../../../../Server/Utils/AI/SRE/Insights/Detectors/MetricDriftDetector.ts"],"names":[],"mappings":"AAAA,OAAO,aAAa,MAAM,0CAA0C,CAAC;AACrE,OAAO,iBAAiB,MAAM,8CAA8C,CAAC;AAC7E,OAAO,qBAAqB,EAAE,EAE5B,qBAAqB,IAAI,0BAA0B,GACpD,MAAM,+CAA+C,CAAC;AAOvD;;;;;;;;;;;;;GAaG;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAW,GAAG,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAW,GAAG,CAAC;AAE5D;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAW,CAAC,CAAC;AAEnD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,4BAA4B,GACvC,0BAA0B,CAAC,yBAAyB,CAAC;AAcvD,MAAM,CAAC,OAAO,OAAO,mBAAmB;IAAxC;QACS,gBAAW,GAAkB,aAAa,CAAC,WAAW,CAAC;IAsJhE,CAAC;IApJC;;;;;;;OAOG;IACI,MAAM,CAAC,aAAa,CACzB,IAAiC;QAEjC,MAAM,YAAY,GAAsC,IAAI,GAAG,EAAE,CAAC;QAClE,MAAM,WAAW,GAAsC,IAAI,GAAG,EAAE,CAAC;QAEjE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,iEAAiE;YACjE,MAAM,OAAO,GAAW,GAAG,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,eAAe,EAAE,CAAC;YAClE,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC5B,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACjC,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAA8B,EAAE,CAAC;QAE/C,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAqC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,SAAS;YACX,CAAC;YACD,IACE,MAAM,CAAC,WAAW,GAAG,mCAAmC;gBACxD,KAAK,CAAC,WAAW,GAAG,mCAAmC,EACvD,CAAC;gBACD,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACrB,SAAS;YACX,CAAC;YAED,MAAM,cAAc,GAClB,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEpD,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,gCAAgC,EAAE,CAAC;gBAChE,SAAS;YACX,CAAC;YAED,QAAQ,CAAC,IAAI,CAAC;gBACZ,UAAU,EAAE,MAAM,CAAC,IAAI;gBACvB,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,UAAU,EAAE,MAAM,CAAC,IAAI;gBACvB,SAAS,EAAE,KAAK,CAAC,IAAI;gBACrB,cAAc;gBACd,iBAAiB,EAAE,MAAM,CAAC,WAAW;gBACrC,gBAAgB,EAAE,KAAK,CAAC,WAAW;aACpC,CAAC,CAAC;QACL,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAqB,EAAE,CAAqB,EAAU,EAAE;YACrE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YACjE,CAAC;YACD,0DAA0D;YAC1D,MAAM,IAAI,GAAW,GAAG,CAAC,CAAC,UAAU,SAAS,CAAC,CAAC,eAAe,EAAE,CAAC;YACjE,MAAM,IAAI,GAAW,GAAG,CAAC,CAAC,UAAU,SAAS,CAAC,CAAC,eAAe,EAAE,CAAC;YACjE,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;IACtD,CAAC;IAED,8CAA8C;IACvC,MAAM,CAAC,mBAAmB,CAAC,cAAsB;QACtD,MAAM,OAAO,GAAW,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC;QACzD,OAAO,GAAG,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,GAAG,CAAC;IACjD,CAAC;IAED,oDAAoD;IAC7C,MAAM,CAAC,gBAAgB,CAC5B,UAAkB,EAClB,eAAuB;QAEvB,OAAO,gBAAgB,UAAU,IAAI,eAAe,EAAE,CAAC;IACzD,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,OAA2B;QAE3B,MAAM,IAAI,GACR,MAAM,qBAAqB,CAAC,oBAAoB,CAAC;YAC/C,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,cAAc,EAAE,4BAA4B;SAC7C,CAAC,CAAC;QAEL,MAAM,QAAQ,GACZ,mBAAmB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAE1C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAA2B,EAAoB,EAAE;YACpE,MAAM,WAAW,GAAW,mBAAmB,CAAC,mBAAmB,CACjE,OAAO,CAAC,cAAc,CACvB,CAAC;YAEF,MAAM,WAAW,GAAkB;gBACjC,4CAA4C;gBAC5C,EAAE;gBACF,eAAe,OAAO,CAAC,UAAU,IAAI;aACtC,CAAC;YACF,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC5B,WAAW,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,eAAe,IAAI,CAAC,CAAC;YAC/D,CAAC;YACD,WAAW,CAAC,IAAI,CACd,wBAAwB,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,iBAAiB,WAAW,EAC9F,uBAAuB,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,gBAAgB,WAAW,EAC3F,sBAAsB,WAAW,EAAE,EACnC,EAAE,EACF,gFAAgF,CACjF,CAAC;YAEF,OAAO;gBACL,WAAW,EAAE,aAAa,CAAC,WAAW;gBACtC,WAAW,EAAE,mBAAmB,CAAC,gBAAgB,CAC/C,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,eAAe,CACxB;gBACD,KAAK,EAAE,iBAAiB,OAAO,CAAC,UAAU,IAAI,WAAW,iBAAiB;gBAC1E,cAAc,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;gBACtC;;;;mBAIG;gBACH,QAAQ,EAAE,iBAAiB,CAAC,GAAG;gBAC/B,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,QAAQ,EAAE;oBACR,WAAW,EAAE;wBACX,UAAU,EAAE,OAAO,CAAC,UAAU;wBAC9B,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,SAAS;wBACrD,cAAc,EAAE,OAAO,CAAC,UAAU;wBAClC,aAAa,EAAE,OAAO,CAAC,SAAS;wBAChC,qBAAqB,EAAE,OAAO,CAAC,cAAc,GAAG,GAAG;wBACnD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;wBAC5C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;qBAC3C;iBACF;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.js

@@ -0,0 +1,177 @@
+import OneUptimeDate from "../../../../../../Types/Date";
+import SortOrder from "../../../../../../Types/BaseDatabase/SortOrder";
+import AIInsightType from "../../../../../../Types/AI/AIInsightType";
+import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
+import TelemetryExceptionService from "../../../../../Services/TelemetryExceptionService";
+import ServiceService from "../../../../../Services/ServiceService";
+import QueryHelper from "../../../../../Types/Database/QueryHelper";
+/*
+ * NewException — a brand-new failure mode in the code: a telemetry
+ * exception group whose FIRST occurrence is inside the lookback window and
+ * that is already recurring (not a one-off blip). Postgres-only and cheap:
+ * the exception groups table carries firstSeenAt and the lifetime
+ * occurrence count, so no ClickHouse scan is needed at all. Deterministic —
+ * no LLM anywhere in this detector.
+ */
+/*
+ * Only exceptions born within this window count as "new". 24h matches the
+ * scanner's dedupe horizon: with a 15-minute scan tick every new exception
+ * is seen many times within the window, and the fingerprint upsert
+ * refreshes the same insight instead of duplicating it.
+ */
+export const NEW_EXCEPTION_LOOKBACK_HOURS = 24;
+/*
+ * A single occurrence is noise (a deploy hiccup, a one-off bad request).
+ * Three occurrences means the failure path is being re-hit — worth a
+ * human's attention.
+ */
+export const NEW_EXCEPTION_MIN_OCCURRENCE_COUNT = 3;
+/*
+ * At 50+ occurrences inside its first day the exception is hammering a hot
+ * path — escalate to High so it sorts above the long tail.
+ */
+export const NEW_EXCEPTION_HIGH_SEVERITY_OCCURRENCE_COUNT = 50;
+/*
+ * Upper bound on candidates per scan tick. Mirrors the TopExceptionsTool
+ * cap — a project melting down does not need 500 insights, and the scanner
+ * additionally caps new insights per tick.
+ */
+export const NEW_EXCEPTION_CANDIDATE_LIMIT = 25;
+// Titles must stay scannable in a list — truncate long exception messages.
+export const NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH = 80;
+export default class NewExceptionDetector {
+    constructor() {
+        this.insightType = AIInsightType.NewException;
+    }
+    /*
+     * Pure decision: does a lifetime occurrence count qualify, and how
+     * urgent is it. The query already filters on the minimum count — this
+     * re-check keeps the rule in one unit-testable place.
+     */
+    static evaluateNewException(occurrenceCount) {
+        return {
+            qualifies: occurrenceCount >= NEW_EXCEPTION_MIN_OCCURRENCE_COUNT,
+            severity: occurrenceCount >= NEW_EXCEPTION_HIGH_SEVERITY_OCCURRENCE_COUNT
+                ? AIInsightSeverity.High
+                : AIInsightSeverity.Medium,
+        };
+    }
+    /*
+     * Human label for the exception: the type when present, else the first
+     * NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH characters of the message.
+     */
+    static buildExceptionLabel(exceptionType, message) {
+        if (exceptionType && exceptionType.trim().length > 0) {
+            return exceptionType.trim();
+        }
+        const trimmedMessage = (message || "").trim();
+        if (trimmedMessage.length === 0) {
+            return "Unknown exception";
+        }
+        if (trimmedMessage.length <= NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH) {
+            return trimmedMessage;
+        }
+        return `${trimmedMessage.slice(0, NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH)}…`;
+    }
+    /*
+     * Stable dedupe key. Keyed on the exception GROUP id (not the message):
+     * the group id is the ingest-side dedupe unit, so the insight refreshes
+     * instead of duplicating across scan ticks.
+     */
+    static buildFingerprint(telemetryExceptionId) {
+        return `new-exception:${telemetryExceptionId}`;
+    }
+    async detect(context) {
+        const firstSeenSince = OneUptimeDate.addRemoveHours(context.now, -1 * NEW_EXCEPTION_LOOKBACK_HOURS);
+        const query = {
+            projectId: context.projectId,
+            firstSeenAt: QueryHelper.greaterThanEqualTo(firstSeenSince),
+            isResolved: false,
+            isArchived: false,
+            occuranceCount: QueryHelper.greaterThanEqualTo(NEW_EXCEPTION_MIN_OCCURRENCE_COUNT),
+        };
+        /*
+         * Root read with an explicit projectId in the query — the scanner is a
+         * server-side worker (monitor-worker precedent), never a user ACL.
+         */
+        const exceptions = await TelemetryExceptionService.findBy({
+            query: query,
+            select: {
+                _id: true,
+                message: true,
+                exceptionType: true,
+                occuranceCount: true,
+                firstSeenAt: true,
+                primaryEntityId: true,
+            },
+            sort: {
+                occuranceCount: SortOrder.Descending,
+            },
+            limit: NEW_EXCEPTION_CANDIDATE_LIMIT,
+            skip: 0,
+            props: { isRoot: true },
+        });
+        const candidates = [];
+        for (const exception of exceptions) {
+            const occurrenceCount = exception.occuranceCount || 0;
+            const decision = NewExceptionDetector.evaluateNewException(occurrenceCount);
+            if (!decision.qualifies || !exception.id) {
+                continue;
+            }
+            const serviceName = await this.resolveServiceName(exception.primaryEntityId);
+            const label = NewExceptionDetector.buildExceptionLabel(exception.exceptionType, exception.message);
+            const firstSeenAtIso = exception.firstSeenAt
+                ? exception.firstSeenAt.toISOString()
+                : undefined;
+            const detailLines = [
+                "**New exception detected**",
+                "",
+                `- Exception: \`${label}\``,
+            ];
+            if (exception.message) {
+                detailLines.push(`- Message: ${exception.message}`);
+            }
+            if (serviceName) {
+                detailLines.push(`- Service: ${serviceName}`);
+            }
+            detailLines.push(`- First seen: ${firstSeenAtIso || "unknown"} (within the last ${NEW_EXCEPTION_LOOKBACK_HOURS}h)`, `- Occurrences so far: ${occurrenceCount}`);
+            candidates.push({
+                insightType: AIInsightType.NewException,
+                fingerprint: NewExceptionDetector.buildFingerprint(exception.id.toString()),
+                title: `New exception: ${label}${serviceName ? ` in ${serviceName}` : ""}`,
+                detailMarkdown: detailLines.join("\n"),
+                severity: decision.severity,
+                serviceName: serviceName || undefined,
+                telemetryServiceId: serviceName ? exception.primaryEntityId : undefined,
+                telemetryExceptionId: exception.id,
+                evidence: {
+                    exception: {
+                        exceptionMessage: exception.message,
+                        exceptionType: exception.exceptionType,
+                        totalOccurrenceCount: occurrenceCount,
+                        firstSeenAt: firstSeenAtIso,
+                    },
+                },
+            });
+        }
+        return candidates;
+    }
+    /*
+     * Best-effort service attribution: primaryEntityId is polymorphic
+     * (Service/Host/DockerHost/...), and findOneById simply returns null for
+     * the non-Service kinds (AIInvestigationAPI precedent) — the insight then
+     * ships without a service name.
+     */
+    async resolveServiceName(primaryEntityId) {
+        if (!primaryEntityId) {
+            return null;
+        }
+        const service = await ServiceService.findOneById({
+            id: primaryEntityId,
+            select: { name: true },
+            props: { isRoot: true },
+        });
+        return (service === null || service === void 0 ? void 0 : service.name) || null;
+    }
+}
+//# sourceMappingURL=NewExceptionDetector.js.map

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NewExceptionDetector.js","sourceRoot":"","sources":["../../../../../../../../Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.ts"],"names":[],"mappings":"AACA,OAAO,aAAa,MAAM,8BAA8B,CAAC;AACzD,OAAO,SAAS,MAAM,gDAAgD,CAAC;AAEvE,OAAO,aAAa,MAAM,0CAA0C,CAAC;AACrE,OAAO,iBAAiB,MAAM,8CAA8C,CAAC;AAG7E,OAAO,yBAAyB,MAAM,mDAAmD,CAAC;AAC1F,OAAO,cAAc,MAAM,wCAAwC,CAAC;AACpE,OAAO,WAAW,MAAM,2CAA2C,CAAC;AAOpE;;;;;;;GAOG;AAEH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAW,EAAE,CAAC;AAEvD;;;;GAIG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAAW,CAAC,CAAC;AAE5D;;;GAGG;AACH,MAAM,CAAC,MAAM,4CAA4C,GAAW,EAAE,CAAC;AAEvE;;;;GAIG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAW,EAAE,CAAC;AAExD,2EAA2E;AAC3E,MAAM,CAAC,MAAM,sCAAsC,GAAW,EAAE,CAAC;AAOjE,MAAM,CAAC,OAAO,OAAO,oBAAoB;IAAzC;QACS,gBAAW,GAAkB,aAAa,CAAC,YAAY,CAAC;IA8KjE,CAAC;IA5KC;;;;OAIG;IACI,MAAM,CAAC,oBAAoB,CAChC,eAAuB;QAEvB,OAAO;YACL,SAAS,EAAE,eAAe,IAAI,kCAAkC;YAChE,QAAQ,EACN,eAAe,IAAI,4CAA4C;gBAC7D,CAAC,CAAC,iBAAiB,CAAC,IAAI;gBACxB,CAAC,CAAC,iBAAiB,CAAC,MAAM;SAC/B,CAAC;IACJ,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,mBAAmB,CAC/B,aAAiC,EACjC,OAA2B;QAE3B,IAAI,aAAa,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,OAAO,aAAa,CAAC,IAAI,EAAE,CAAC;QAC9B,CAAC;QACD,MAAM,cAAc,GAAW,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,mBAAmB,CAAC;QAC7B,CAAC;QACD,IAAI,cAAc,CAAC,MAAM,IAAI,sCAAsC,EAAE,CAAC;YACpE,OAAO,cAAc,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,sCAAsC,CAAC,GAAG,CAAC;IAC/E,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,gBAAgB,CAAC,oBAA4B;QACzD,OAAO,iBAAiB,oBAAoB,EAAE,CAAC;IACjD,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,OAA2B;QAE3B,MAAM,cAAc,GAAS,aAAa,CAAC,cAAc,CACvD,OAAO,CAAC,GAAG,EACX,CAAC,CAAC,GAAG,4BAA4B,CAClC,CAAC;QAEF,MAAM,KAAK,GAAe;YACxB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,WAAW,EAAE,WAAW,CAAC,kBAAkB,CAAC,cAAc,CAAC;YAC3D,UAAU,EAAE,KAAK;YACjB,UAAU,EAAE,KAAK;YACjB,cAAc,EAAE,WAAW,CAAC,kBAAkB,CAC5C,kCAAkC,CACnC;SACF,CAAC;QAEF;;;WAGG;QACH,MAAM,UAAU,GACd,MAAM,yBAAyB,CAAC,MAAM,CAAC;YACrC,KAAK,EAAE,KAAc;YACrB,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI;gBACT,OAAO,EAAE,IAAI;gBACb,aAAa,EAAE,IAAI;gBACnB,cAAc,EAAE,IAAI;gBACpB,WAAW,EAAE,IAAI;gBACjB,eAAe,EAAE,IAAI;aACtB;YACD,IAAI,EAAE;gBACJ,cAAc,EAAE,SAAS,CAAC,UAAU;aACrC;YACD,KAAK,EAAE,6BAA6B;YACpC,IAAI,EAAE,CAAC;YACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QAEL,MAAM,UAAU,GAA4B,EAAE,CAAC;QAE/C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,eAAe,GAAW,SAAS,CAAC,cAAc,IAAI,CAAC,CAAC;YAC9D,MAAM,QAAQ,GACZ,oBAAoB,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;YAE7D,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,WAAW,GAAkB,MAAM,IAAI,CAAC,kBAAkB,CAC9D,SAAS,CAAC,eAAe,CAC1B,CAAC;YAEF,MAAM,KAAK,GAAW,oBAAoB,CAAC,mBAAmB,CAC5D,SAAS,CAAC,aAAa,EACvB,SAAS,CAAC,OAAO,CAClB,CAAC;YAEF,MAAM,cAAc,GAAuB,SAAS,CAAC,WAAW;gBAC9D,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,WAAW,EAAE;gBACrC,CAAC,CAAC,SAAS,CAAC;YAEd,MAAM,WAAW,GAAkB;gBACjC,4BAA4B;gBAC5B,EAAE;gBACF,kBAAkB,KAAK,IAAI;aAC5B,CAAC;YACF,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;gBACtB,WAAW,CAAC,IAAI,CAAC,cAAc,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,WAAW,EAAE,CAAC;gBAChB,WAAW,CAAC,IAAI,CAAC,cAAc,WAAW,EAAE,CAAC,CAAC;YAChD,CAAC;YACD,WAAW,CAAC,IAAI,CACd,iBAAiB,cAAc,IAAI,SAAS,qBAAqB,4BAA4B,IAAI,EACjG,yBAAyB,eAAe,EAAE,CAC3C,CAAC;YAEF,UAAU,CAAC,IAAI,CAAC;gBACd,WAAW,EAAE,aAAa,CAAC,YAAY;gBACvC,WAAW,EAAE,oBAAoB,CAAC,gBAAgB,CAChD,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,CACxB;gBACD,KAAK,EAAE,kBAAkB,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC1E,cAAc,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;gBACtC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,WAAW,EAAE,WAAW,IAAI,SAAS;gBACrC,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;gBACvE,oBAAoB,EAAE,SAAS,CAAC,EAAE;gBAClC,QAAQ,EAAE;oBACR,SAAS,EAAE;wBACT,gBAAgB,EAAE,SAAS,CAAC,OAAO;wBACnC,aAAa,EAAE,SAAS,CAAC,aAAa;wBACtC,oBAAoB,EAAE,eAAe;wBACrC,WAAW,EAAE,cAAc;qBAC5B;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,kBAAkB,CAC9B,eAAqC;QAErC,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAAmB,MAAM,cAAc,CAAC,WAAW,CAAC;YAC/D,EAAE,EAAE,eAAe;YACnB,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;YACtB,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QACH,OAAO,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,KAAI,IAAI,CAAC;IAC/B,CAAC;CACF"}