@oneuptime/common 11.4.0 → 11.5.0

package/Server/Services/TelemetryEntityService.ts

@@ -1,5 +1,9 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/TelemetryEntity";
+import Service from "../../Models/DatabaseModels/Service";
+import ServiceService from "./ServiceService";
+import QueryHelper from "../Types/Database/QueryHelper";
+import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import ColumnLength from "../../Types/Database/ColumnLength";
@@ -68,6 +72,51 @@ export class TelemetryEntityService extends DatabaseService<Model> {
     }
   }
 
+  /**
+   * The polymorphic (resourceType, resourceId) pointer for Service
+   * entities: resolve the backing Service row by name (case-insensitive —
+   * entity display names are canonicalized to lowercase while Service
+   * rows keep their original casing, the same matching ingest itself
+   * uses). This is the single place the name join happens; everything
+   * downstream (traces links, overlays) reads the pointer instead of
+   * re-matching names. Best-effort: no Service row means no pointer.
+   */
+  private async resolveServiceResourceId(data: {
+    projectId: ObjectID;
+    entity: ExtractedEntity;
+  }): Promise<ObjectID | null> {
+    if (data.entity.entityType !== EntityType.Service) {
+      return null;
+    }
+    const name: string = TelemetryEntityService.deriveDisplayName(data.entity);
+    if (!name) {
+      return null;
+    }
+    try {
+      /*
+       * createdAt ASC mirrors ingest's findOrCreateTelemetryService: if
+       * case-variant duplicate Service rows exist, telemetry keys to the
+       * OLDEST row, so the pointer must resolve to the same one.
+       */
+      const service: Service | null = await ServiceService.findOneBy({
+        query: {
+          projectId: data.projectId,
+          name: QueryHelper.findWithSameText(name),
+        },
+        select: { _id: true },
+        sort: { createdAt: SortOrder.Ascending },
+        props: { isRoot: true },
+      });
+      return service?._id ? new ObjectID(service._id.toString()) : null;
+    } catch (err) {
+      logger.error(
+        `TelemetryEntityService: failed to resolve Service row for entity name "${name}":`,
+      );
+      logger.error(err as Error);
+      return null;
+    }
+  }
+
   private async upsertEntity(data: {
     projectId: ObjectID;
     entity: ExtractedEntity;
@@ -76,6 +125,9 @@ export class TelemetryEntityService extends DatabaseService<Model> {
     const { projectId, entity, countByType } = data;
     const now: Date = OneUptimeDate.getCurrentDate();
 
+    const serviceResourceId: ObjectID | null =
+      await this.resolveServiceResourceId({ projectId, entity });
+
     await reconcileByNaturalKey({
       service: this,
       query: {
@@ -85,9 +137,23 @@ export class TelemetryEntityService extends DatabaseService<Model> {
       },
       lastSeenAt: now,
       describe: `entity ${entity.entityType}/${entity.entityKey}`,
-      select: { descriptiveAttributes: true, labels: true },
+      select: { descriptiveAttributes: true, labels: true, resourceId: true },
       buildUpdate: (existing: Model): QueryDeepPartialEntity<Model> => {
-        return TelemetryEntityService.buildDescriptiveUpdate(entity, existing);
+        const update: QueryDeepPartialEntity<Model> =
+          TelemetryEntityService.buildDescriptiveUpdate(entity, existing);
+        /*
+         * Stamp / refresh the pointer on the throttled bump too, so
+         * pre-existing rows converge and a deleted+recreated Service
+         * (new id, same name) heals instead of pointing at a dead row.
+         */
+        if (
+          serviceResourceId &&
+          existing.resourceId?.toString() !== serviceResourceId.toString()
+        ) {
+          (update as { resourceType?: string }).resourceType = "Service";
+          (update as { resourceId?: ObjectID }).resourceId = serviceResourceId;
+        }
+        return update;
       },
       /*
        * Entity budget (doc §Edge Cases): existing rows always get their
@@ -145,6 +211,10 @@ export class TelemetryEntityService extends DatabaseService<Model> {
         model.displayName = TelemetryEntityService.deriveDisplayName(
           entity,
         ).substring(0, ColumnLength.ShortText);
+        if (serviceResourceId) {
+          model.resourceType = "Service";
+          model.resourceId = serviceResourceId;
+        }
         model.firstSeenAt = now;
         model.lastSeenAt = now;
         return model;

package/Server/Utils/AI/Sentinel/AlertInvestigationRunner.ts

@@ -1,12 +1,23 @@
 import ObjectID from "../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../Types/Date";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import AIRunType from "../../../../Types/AI/AIRunType";
 import { Blue500 } from "../../../../Types/BrandColors";
+import Alert from "../../../../Models/DatabaseModels/Alert";
+import AlertSeverity from "../../../../Models/DatabaseModels/AlertSeverity";
+import Project from "../../../../Models/DatabaseModels/Project";
 import { AlertFeedEventType } from "../../../../Models/DatabaseModels/AlertFeed";
+import AlertService from "../../../Services/AlertService";
+import AlertSeverityService from "../../../Services/AlertSeverityService";
+import ProjectService from "../../../Services/ProjectService";
+import AIRunService from "../../../Services/AIRunService";
 import AlertFeedService from "../../../Services/AlertFeedService";
+import QueryHelper from "../../../Types/Database/QueryHelper";
 import AlertAIContextBuilder, {
   AlertContextData,
 } from "../AlertAIContextBuilder";
 import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
+import SentinelInvestigationQueue from "./InvestigationQueue";
 import { ObservabilityAssistantResult } from "../Chat/ObservabilityAssistant";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
@@ -19,13 +30,33 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  * the alert timeline + Slack/Teams. Symmetric to incident investigation and
  * built on the same shared SentinelInvestigationEngine.
  *
- * Gated by the same per-project opt-in as incidents. Alert volume can be higher
- * than incidents, so severity / rate gating is a deliberate follow-up.
+ * Gated by the same per-project opt-in as incidents, PLUS alert-specific cost
+ * gates (alert volume can be far higher than incidents):
+ *   - severity floor: only alerts at or above the project's configured minimum
+ *     severity are investigated; when unset, the top two severity tiers
+ *     (lowest two `order` values) are the default.
+ *   - per-monitor dedupe window: a monitor that already triggered an
+ *     investigation recently is not re-investigated — the first RCA stands.
  */
+
+// Per-project override lives in Project.alertInvestigationDedupeWindowMinutes.
+export const DEFAULT_DEDUPE_WINDOW_MINUTES: number = 30;
+// Clamp: an RCA older than a day is a new episode by any reading.
+const MAX_DEDUPE_WINDOW_MINUTES: number = 24 * 60;
+
+export interface AlertGateDecision {
+  investigate: boolean;
+  // Human-readable reason recorded in the debug log when skipping.
+  reason: string;
+  // Passed through to the AIRun as the dedupe key for future windows.
+  monitorId?: ObjectID | undefined;
+}
+
 export default class SentinelAlertInvestigationRunner {
   /*
    * Entry point called (fire-and-forget) from AlertService.onCreateSuccess.
-   * Never throws — all failures are logged and recorded on the AIRun.
+   * Cheap: gates + a Queued AIRun row; execution happens via the queue.
+   * Never throws — all failures are logged.
    */
   @CaptureSpan()
   public static async investigateNewAlert(data: {
@@ -44,16 +75,76 @@ export default class SentinelAlertInvestigationRunner {
         return;
       }
 
+      const gate: AlertGateDecision = await this.shouldInvestigateAlert({
+        alertId,
+        projectId,
+      });
+
+      if (!gate.investigate) {
+        logger.debug(
+          `Sentinel: skipping investigation for alert ${alertId.toString()} — ${gate.reason}.`,
+        );
+        return;
+      }
+
+      await SentinelInvestigationQueue.enqueue({
+        projectId,
+        subjectAlertId: alertId,
+        subjectMonitorId: gate.monitorId,
+      });
+    } catch (error) {
+      logger.error(
+        `Sentinel: unexpected error enqueueing investigation for alert ${alertId.toString()}: ${error}`,
+      );
+    }
+  }
+
+  /*
+   * Execute a claimed run: assemble alert context and hand it to the engine.
+   * Called by SentinelInvestigationQueue after a successful claim. Never
+   * throws — failures are handed to the queue's retry policy so a claimed
+   * run is always finalized or requeued.
+   */
+  @CaptureSpan()
+  public static async executeInvestigation(data: {
+    aiRunId: ObjectID;
+    projectId: ObjectID;
+    alertId: ObjectID;
+    attemptCount: number;
+  }): Promise<void> {
+    const { aiRunId, projectId, alertId, attemptCount } = data;
+
+    let contextSummary: string;
+    try {
       const contextData: AlertContextData =
         await AlertAIContextBuilder.buildAlertContext({
           alertId,
         });
 
-      await SentinelInvestigationEngine.investigate({
-        projectId,
+      contextSummary = this.buildAlertSummary(contextData);
+    } catch (error) {
+      /*
+       * Context assembly failed — the run is claimed, so hand it to the
+       * retry policy rather than leaving it Running until the sweeper.
+       */
+      await SentinelInvestigationQueue.failOrRequeue({
+        aiRunId,
+        attemptCount,
+        errorMessage: `Failed to build alert context: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+        isPermanent: false,
+      });
+      return;
+    }
+
+    await SentinelInvestigationEngine.executeRun({
+      aiRunId,
+      projectId,
+      attemptCount,
+      request: {
         feature: "Sentinel Alert Investigation",
-        subjectAlertId: alertId,
-        contextSummary: this.buildAlertSummary(contextData),
+        contextSummary,
         postAnalysis: async (postData: {
           analysisMarkdown: string;
           isConfident: boolean;
@@ -74,12 +165,153 @@ export default class SentinelAlertInvestigationRunner {
             },
           });
         },
-      });
-    } catch (error) {
-      logger.error(
-        `Sentinel: unexpected error investigating alert ${alertId.toString()}: ${error}`,
-      );
+      },
+    });
+  }
+
+  /*
+   * The alert-specific cost gates, evaluated BEFORE the expensive context
+   * build. Severity first (cheapest to reason about), then the per-monitor
+   * dedupe window. The severity gate only filters KNOWN-low-severity noise:
+   * an alert whose severity order cannot be determined passes it.
+   */
+  @CaptureSpan()
+  public static async shouldInvestigateAlert(data: {
+    alertId: ObjectID;
+    projectId: ObjectID;
+  }): Promise<AlertGateDecision> {
+    const { alertId, projectId } = data;
+
+    const alert: Alert | null = await AlertService.findOneById({
+      id: alertId,
+      select: {
+        monitorId: true,
+        alertSeverity: {
+          order: true,
+        },
+      },
+      props: { isRoot: true },
+    });
+
+    if (!alert) {
+      return { investigate: false, reason: "alert not found" };
     }
+
+    // One project read serves both the severity floor and the dedupe window.
+    const project: Project | null = await ProjectService.findOneById({
+      id: projectId,
+      select: {
+        alertInvestigationMinimumSeverityId: true,
+        alertInvestigationDedupeWindowMinutes: true,
+      },
+      props: { isRoot: true },
+    });
+
+    // Severity floor.
+    const floorOrder: number | null = await this.getSeverityFloorOrder(
+      projectId,
+      project?.alertInvestigationMinimumSeverityId,
+    );
+    const alertOrder: number | undefined = alert.alertSeverity?.order;
+
+    if (
+      floorOrder !== null &&
+      alertOrder !== undefined &&
+      alertOrder > floorOrder
+    ) {
+      return {
+        investigate: false,
+        reason: `severity order ${alertOrder} is below the investigation floor (order ${floorOrder})`,
+        monitorId: alert.monitorId,
+      };
+    }
+
+    /*
+     * Per-monitor dedupe window: per-project override, defaulting to 30
+     * minutes, clamped to at most a day; 0 disables the cooldown. Alerts
+     * without a monitor have no dedupe key.
+     */
+    const dedupeWindowMinutes: number = Math.min(
+      MAX_DEDUPE_WINDOW_MINUTES,
+      Math.max(
+        0,
+        project?.alertInvestigationDedupeWindowMinutes ??
+          DEFAULT_DEDUPE_WINDOW_MINUTES,
+      ),
+    );
+
+    if (alert.monitorId && dedupeWindowMinutes > 0) {
+      const windowStart: Date =
+        OneUptimeDate.getSomeMinutesAgo(dedupeWindowMinutes);
+
+      const recentRunCount: number = (
+        await AIRunService.countBy({
+          query: {
+            projectId,
+            monitorId: alert.monitorId,
+            runType: AIRunType.Investigation,
+            createdAt: QueryHelper.greaterThan(windowStart),
+          },
+          props: { isRoot: true },
+        })
+      ).toNumber();
+
+      if (recentRunCount > 0) {
+        return {
+          investigate: false,
+          reason: `monitor ${alert.monitorId.toString()} was already investigated within the last ${dedupeWindowMinutes} minutes`,
+          monitorId: alert.monitorId,
+        };
+      }
+    }
+
+    return {
+      investigate: true,
+      reason: "passed severity and dedupe gates",
+      monitorId: alert.monitorId,
+    };
+  }
+
+  /*
+   * The severity `order` value that still qualifies for investigation (lower
+   * order = higher severity; an alert qualifies when its order <= floor).
+   * An explicitly configured minimum severity wins; when unset — or when the
+   * configured severity has been deleted — the default is the project's top
+   * two tiers. Returns null when no floor applies (no severities configured).
+   */
+  private static async getSeverityFloorOrder(
+    projectId: ObjectID,
+    minimumSeverityId: ObjectID | undefined,
+  ): Promise<number | null> {
+    if (minimumSeverityId) {
+      const floorSeverity: AlertSeverity | null =
+        await AlertSeverityService.findOneById({
+          id: minimumSeverityId,
+          select: { order: true },
+          props: { isRoot: true },
+        });
+
+      if (floorSeverity && floorSeverity.order !== undefined) {
+        return floorSeverity.order;
+      }
+    }
+
+    // Default: the top two severity tiers (the two lowest order values).
+    const topTiers: Array<AlertSeverity> = await AlertSeverityService.findBy({
+      query: { projectId },
+      select: { order: true },
+      sort: { order: SortOrder.Ascending },
+      limit: 2,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    if (topTiers.length === 0) {
+      return null;
+    }
+
+    const lastTier: AlertSeverity = topTiers[topTiers.length - 1]!;
+    return lastTier.order ?? null;
   }
 
   // Build a compact alert record to seed the investigation.

package/Server/Utils/AI/Sentinel/IncidentInvestigationRunner.ts

@@ -2,11 +2,14 @@ import ObjectID from "../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../Types/Date";
 import { Blue500 } from "../../../../Types/BrandColors";
 import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
+import IncidentInternalNote from "../../../../Models/DatabaseModels/IncidentInternalNote";
 import IncidentFeedService from "../../../Services/IncidentFeedService";
+import IncidentInternalNoteService from "../../../Services/IncidentInternalNoteService";
 import IncidentAIContextBuilder, {
   IncidentContextData,
 } from "../IncidentAIContextBuilder";
 import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
+import SentinelInvestigationQueue from "./InvestigationQueue";
 import SentinelMemory from "./SentinelMemory";
 import { ObservabilityAssistantResult } from "../Chat/ObservabilityAssistant";
 import logger from "../../Logger";
@@ -17,18 +20,17 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  *
  * When a new incident is declared, wakes Sentinel to investigate it (read-only)
  * and post a cited root-cause analysis into the incident timeline + Slack/Teams
- * — before the on-call engineer has finished reading the page. The heavy lifting
- * (run lifecycle, the agent loop, confidence gating) lives in the shared
- * SentinelInvestigationEngine; this file only assembles incident context and
- * posts the result to the incident's feed.
- *
- * Built on the fire-and-forget async pattern already used throughout
- * IncidentService.onCreateSuccess. See Internal/Roadmap/AISentinelReliabilityBrain.md.
+ * — before the on-call engineer has finished reading the page. The trigger only
+ * records durable intent (a Queued AIRun via SentinelInvestigationQueue), so a
+ * pod restart can no longer orphan an investigation; the heavy lifting (run
+ * lifecycle, the agent loop, confidence gating) lives in the shared
+ * SentinelInvestigationEngine. See Internal/Roadmap/AISentinelExecution.md.
  */
 export default class SentinelIncidentInvestigationRunner {
   /*
    * Entry point called (fire-and-forget) from IncidentService.onCreateSuccess.
-   * Never throws — all failures are logged and recorded on the AIRun.
+   * Cheap: gates + a Queued AIRun row; execution happens via the queue.
+   * Never throws — all failures are logged.
    */
   @CaptureSpan()
   public static async investigateNewIncident(data: {
@@ -47,6 +49,34 @@ export default class SentinelIncidentInvestigationRunner {
         return;
       }
 
+      await SentinelInvestigationQueue.enqueue({
+        projectId,
+        subjectIncidentId: incidentId,
+      });
+    } catch (error) {
+      logger.error(
+        `Sentinel: unexpected error enqueueing investigation for incident ${incidentId.toString()}: ${error}`,
+      );
+    }
+  }
+
+  /*
+   * Execute a claimed run: assemble incident context and hand it to the
+   * engine. Called by SentinelInvestigationQueue after a successful claim.
+   * Never throws — failures are handed to the queue's retry policy so a
+   * claimed run is always finalized or requeued.
+   */
+  @CaptureSpan()
+  public static async executeInvestigation(data: {
+    aiRunId: ObjectID;
+    projectId: ObjectID;
+    incidentId: ObjectID;
+    attemptCount: number;
+  }): Promise<void> {
+    const { aiRunId, projectId, incidentId, attemptCount } = data;
+
+    let contextSummary: string;
+    try {
       const contextData: IncidentContextData =
         await IncidentAIContextBuilder.buildIncidentContext({
           incidentId,
@@ -74,12 +104,31 @@ export default class SentinelIncidentInvestigationRunner {
             .filter(Boolean),
         });
 
-      await SentinelInvestigationEngine.investigate({
-        projectId,
+      contextSummary =
+        this.buildIncidentSummary(contextData) + priorCasesContext;
+    } catch (error) {
+      /*
+       * Context assembly failed — the run is claimed, so hand it to the
+       * retry policy rather than leaving it Running until the sweeper.
+       */
+      await SentinelInvestigationQueue.failOrRequeue({
+        aiRunId,
+        attemptCount,
+        errorMessage: `Failed to build incident context: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+        isPermanent: false,
+      });
+      return;
+    }
+
+    await SentinelInvestigationEngine.executeRun({
+      aiRunId,
+      projectId,
+      attemptCount,
+      request: {
         feature: "Sentinel Incident Investigation",
-        subjectIncidentId: incidentId,
-        contextSummary:
-          this.buildIncidentSummary(contextData) + priorCasesContext,
+        contextSummary,
         postAnalysis: async (postData: {
           analysisMarkdown: string;
           isConfident: boolean;
@@ -99,13 +148,37 @@ export default class SentinelIncidentInvestigationRunner {
               sendWorkspaceNotification: postData.isConfident,
             },
           });
+
+          /*
+           * Also file the RCA as an incident internal note, where responders
+           * collaborate. The RootCause feed item above, with its quiet-mode
+           * gating, must stay the single notification source, so:
+           * ignoreHooks skips the note service's "posted private note" feed
+           * item + unconditional workspace ping (workflow triggers and
+           * realtime updates still fire), and isOwnerNotified stops the
+           * note-posted owner-notification cron from paging owners about it.
+           * Best-effort: a note failure must not fail the run after the feed
+           * item already posted.
+           */
+          try {
+            const note: IncidentInternalNote = new IncidentInternalNote();
+            note.incidentId = incidentId;
+            note.projectId = projectId;
+            note.note = postData.analysisMarkdown;
+            note.isOwnerNotified = true;
+
+            await IncidentInternalNoteService.create({
+              data: note,
+              props: { isRoot: true, ignoreHooks: true },
+            });
+          } catch (error) {
+            logger.error(
+              `Sentinel: failed to post RCA internal note for incident ${incidentId.toString()}: ${error}`,
+            );
+          }
         },
-      });
-    } catch (error) {
-      logger.error(
-        `Sentinel: unexpected error investigating incident ${incidentId.toString()}: ${error}`,
-      );
-    }
+      },
+    });
   }
 
   // Build a compact incident record to seed the investigation.