@oneuptime/common 11.3.28 → 11.4.0

package/Models/DatabaseModels/AIRun.ts

@@ -284,6 +284,56 @@ export default class AIRun extends BaseModel {
   })
   public conversationId?: ObjectID = undefined;
 
+  @ColumnAccessControl({
+    create: [],
+    read: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+    ],
+    update: [],
+  })
+  @Index()
+  @TableColumn({
+    type: TableColumnType.ObjectID,
+    required: false,
+    canReadOnRelationQuery: true,
+    title: "Triggered By Incident ID",
+    description:
+      "The incident that triggered this run (for autonomous investigations).",
+  })
+  @Column({
+    type: ColumnType.ObjectID,
+    nullable: true,
+    transformer: ObjectID.getDatabaseTransformer(),
+  })
+  public triggeredByIncidentId?: ObjectID = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+    ],
+    update: [],
+  })
+  @Index()
+  @TableColumn({
+    type: TableColumnType.ObjectID,
+    required: false,
+    canReadOnRelationQuery: true,
+    title: "Triggered By Alert ID",
+    description:
+      "The alert that triggered this run (for autonomous investigations).",
+  })
+  @Column({
+    type: ColumnType.ObjectID,
+    nullable: true,
+    transformer: ObjectID.getDatabaseTransformer(),
+  })
+  public triggeredByAlertId?: ObjectID = undefined;
+
   @ColumnAccessControl({
     create: [],
     read: [

package/Models/DatabaseModels/Project.ts

@@ -1422,6 +1422,66 @@ export default class Project extends TenantModel {
   })
   public enableAi?: boolean = undefined;
 
+  @ColumnAccessControl({
+    create: [],
+    read: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.Viewer,
+      Permission.ReadProject,
+      Permission.UnAuthorizedSsoUser,
+      Permission.ProjectUser,
+    ],
+    update: [Permission.ProjectOwner, Permission.ProjectAdmin],
+  })
+  @TableColumn({
+    required: true,
+    isDefaultValueColumn: true,
+    type: TableColumnType.Boolean,
+    title: "Enable Automatic Incident Investigation",
+    description:
+      "When enabled, OneUptime's AI SRE (Sentinel) automatically investigates every new incident and posts a cited root cause analysis to the incident timeline. Requires AI to be enabled and an LLM provider to be configured.",
+    defaultValue: false,
+    example: true,
+  })
+  @Column({
+    nullable: false,
+    default: false,
+    type: ColumnType.Boolean,
+  })
+  public enableAutomaticIncidentInvestigation?: boolean = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.Viewer,
+      Permission.ReadProject,
+      Permission.UnAuthorizedSsoUser,
+      Permission.ProjectUser,
+    ],
+    update: [Permission.ProjectOwner, Permission.ProjectAdmin],
+  })
+  @TableColumn({
+    required: true,
+    isDefaultValueColumn: true,
+    type: TableColumnType.Boolean,
+    title: "Enable Automatic Alert Investigation",
+    description:
+      "When enabled, OneUptime's AI SRE (Sentinel) automatically investigates every new alert and posts a cited root cause analysis to the alert timeline. Requires AI to be enabled and an LLM provider to be configured.",
+    defaultValue: false,
+    example: true,
+  })
+  @Column({
+    nullable: false,
+    default: false,
+    type: ColumnType.Boolean,
+  })
+  public enableAutomaticAlertInvestigation?: boolean = undefined;
+
   @ColumnAccessControl({
     create: [],
     read: [

package/Server/API/AIInvestigationAPI.ts

@@ -0,0 +1,148 @@
+import UserMiddleware from "../Middleware/UserAuthorization";
+import CommonAPI from "./CommonAPI";
+import Express, {
+  ExpressRequest,
+  ExpressResponse,
+  ExpressRouter,
+  NextFunction,
+} from "../Utils/Express";
+import Response from "../Utils/Response";
+import DatabaseCommonInteractionProps from "../../Types/BaseDatabase/DatabaseCommonInteractionProps";
+import ObjectID from "../../Types/ObjectID";
+import BadDataException from "../../Types/Exception/BadDataException";
+import NotAuthorizedException from "../../Types/Exception/NotAuthorizedException";
+import SortOrder from "../../Types/BaseDatabase/SortOrder";
+import { JSONArray, JSONObject } from "../../Types/JSON";
+import AIRunType from "../../Types/AI/AIRunType";
+import AIRun from "../../Models/DatabaseModels/AIRun";
+import AIRunEvent from "../../Models/DatabaseModels/AIRunEvent";
+import Incident from "../../Models/DatabaseModels/Incident";
+import BaseModel from "../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
+import IncidentService from "../Services/IncidentService";
+import AIRunService from "../Services/AIRunService";
+import AIRunEventService from "../Services/AIRunEventService";
+
+const router: ExpressRouter = Express.getRouter();
+
+const MAX_EVENTS: number = 500;
+
+/*
+ * Returns the latest Sentinel investigation (the AIRun + its ordered
+ * AIRunEvents) for an incident, so the dashboard can render a live "watch it
+ * think" panel.
+ *
+ * Investigation runs are system-authored (userId = null) and are therefore
+ * hidden by the per-user privacy pin on the generic AIRun / AIRunEvent CRUD. So
+ * we gate access explicitly here: first confirm the caller can read the incident
+ * under THEIR permissions, then read the run + events as root.
+ */
+router.post(
+  "/ai-investigation/incident",
+  UserMiddleware.getUserMiddleware,
+  async (
+    req: ExpressRequest,
+    res: ExpressResponse,
+    next: NextFunction,
+  ): Promise<void> => {
+    try {
+      const props: DatabaseCommonInteractionProps =
+        await CommonAPI.getDatabaseCommonInteractionProps(req);
+
+      if (!props.userId) {
+        throw new NotAuthorizedException(
+          "A logged-in user session is required.",
+        );
+      }
+
+      const incidentIdString: string | undefined = req.body["incidentId"] as
+        | string
+        | undefined;
+
+      if (!incidentIdString) {
+        throw new BadDataException("incidentId is required.");
+      }
+
+      const incidentId: ObjectID = new ObjectID(incidentIdString);
+
+      // Access check under the USER's permissions (null when not allowed).
+      const incident: Incident | null = await IncidentService.findOneById({
+        id: incidentId,
+        select: { _id: true },
+        props,
+      });
+
+      if (!incident) {
+        throw new BadDataException(
+          "Incident not found (or you do not have access to it).",
+        );
+      }
+
+      // Read the latest investigation run as root (bypasses the per-user pin).
+      const runs: Array<AIRun> = await AIRunService.findBy({
+        query: {
+          triggeredByIncidentId: incidentId,
+          runType: AIRunType.Investigation,
+        },
+        select: {
+          _id: true,
+          status: true,
+          startedAt: true,
+          completedAt: true,
+          errorMessage: true,
+          llmCallCount: true,
+          toolCallCount: true,
+          totalTokens: true,
+          createdAt: true,
+        },
+        sort: { createdAt: SortOrder.Descending },
+        limit: 1,
+        skip: 0,
+        props: { isRoot: true },
+      });
+
+      const run: AIRun | undefined = runs[0];
+
+      if (!run) {
+        Response.sendJsonObjectResponse(req, res, {
+          run: null,
+          events: [],
+        });
+        return;
+      }
+
+      const events: Array<AIRunEvent> = await AIRunEventService.findBy({
+        query: { aiRunId: run.id! },
+        select: {
+          _id: true,
+          sequence: true,
+          eventType: true,
+          toolName: true,
+          resultSummary: true,
+          createdAt: true,
+        },
+        sort: { sequence: SortOrder.Ascending },
+        limit: MAX_EVENTS,
+        skip: 0,
+        props: { isRoot: true },
+      });
+
+      const runJson: JSONObject | undefined = BaseModel.toJSONArray(
+        [run],
+        AIRun,
+      )[0];
+
+      const eventsJson: JSONArray = BaseModel.toJSONArray(events, AIRunEvent);
+
+      Response.sendJsonObjectResponse(req, res, {
+        run: runJson || null,
+        events: eventsJson,
+      });
+      return;
+    } catch (err) {
+      next(err);
+      return;
+    }
+  },
+);
+
+export default router;

package/Server/Infrastructure/Postgres/SchemaMigrations/1783598145111-AddEnableAutomaticIncidentInvestigationToProject.ts

@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddEnableAutomaticIncidentInvestigationToProject1783598145111
+  implements MigrationInterface
+{
+  public name = "AddEnableAutomaticIncidentInvestigationToProject1783598145111";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableAutomaticIncidentInvestigation" boolean NOT NULL DEFAULT false`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableAutomaticIncidentInvestigation"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1783615884106-AddInvestigationSubjectToAIRun.ts

@@ -0,0 +1,33 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddInvestigationSubjectToAIRun1783615884106
+  implements MigrationInterface
+{
+  public name = "AddInvestigationSubjectToAIRun1783615884106";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" ADD "triggeredByIncidentId" uuid`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" ADD "triggeredByAlertId" uuid`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_AIRun_triggeredByIncidentId" ON "AIRun" ("triggeredByIncidentId")`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_AIRun_triggeredByAlertId" ON "AIRun" ("triggeredByAlertId")`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "IDX_AIRun_triggeredByAlertId"`);
+    await queryRunner.query(`DROP INDEX "IDX_AIRun_triggeredByIncidentId"`);
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" DROP COLUMN "triggeredByAlertId"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" DROP COLUMN "triggeredByIncidentId"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1783632447975-AddEnableAutomaticAlertInvestigationToProject.ts

@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddEnableAutomaticAlertInvestigationToProject1783632447975
+  implements MigrationInterface
+{
+  public name = "AddEnableAutomaticAlertInvestigationToProject1783632447975";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableAutomaticAlertInvestigation" boolean NOT NULL DEFAULT false`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableAutomaticAlertInvestigation"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -417,6 +417,9 @@ import { AddIncomingCallPolicyRoutingPhoneNumberUnique1783470000000 } from "./17
 import { FixNotNullForeignKeysOnDelete1783510935686 } from "./1783510935686-FixNotNullForeignKeysOnDelete";
 import { MigrationName1783515836148 } from "./1783515836148-MigrationName";
 import { AddMigrationFailureTable1783523076215 } from "./1783523076215-AddMigrationFailureTable";
+import { AddEnableAutomaticIncidentInvestigationToProject1783598145111 } from "./1783598145111-AddEnableAutomaticIncidentInvestigationToProject";
+import { AddInvestigationSubjectToAIRun1783615884106 } from "./1783615884106-AddInvestigationSubjectToAIRun";
+import { AddEnableAutomaticAlertInvestigationToProject1783632447975 } from "./1783632447975-AddEnableAutomaticAlertInvestigationToProject";
 
 export default [
   InitialMigration,
@@ -838,4 +841,7 @@ export default [
   FixNotNullForeignKeysOnDelete1783510935686,
   MigrationName1783515836148,
   AddMigrationFailureTable1783523076215,
+  AddEnableAutomaticIncidentInvestigationToProject1783598145111,
+  AddInvestigationSubjectToAIRun1783615884106,
+  AddEnableAutomaticAlertInvestigationToProject1783632447975,
 ];

package/Server/Services/AlertService.ts

@@ -67,6 +67,7 @@ import AlertLabelRuleEngineService from "./AlertLabelRuleEngineService";
 import AlertOnCallRuleEngineService from "./AlertOnCallRuleEngineService";
 import AlertOwnerRuleEngineService from "./AlertOwnerRuleEngineService";
 import RunbookRuleEngineService from "./RunbookRuleEngineService";
+import SentinelAlertInvestigationRunner from "../Utils/AI/Sentinel/AlertInvestigationRunner";
 import AlertPrivacyRuleEngineService from "./AlertPrivacyRuleEngineService";
 import ProjectService from "./ProjectService";
 
@@ -558,6 +559,30 @@ export class Service extends DatabaseService<Model> {
           );
         }
       })
+      .then(async () => {
+        /*
+         * Sentinel (AI SRE): automatically investigate the new alert and post a
+         * cited root cause analysis to the alert timeline + Slack/Teams. Runs
+         * last, is gated per project (opt-in) + requires an LLM provider, and is
+         * read-only.
+         */
+        try {
+          if (createdItem.projectId && createdItem.id) {
+            await SentinelAlertInvestigationRunner.investigateNewAlert({
+              alertId: createdItem.id,
+              projectId: createdItem.projectId,
+            });
+          }
+        } catch (error) {
+          logger.error(
+            `Sentinel alert investigation failed in AlertService.onCreateSuccess: ${error}`,
+            {
+              projectId: createdItem.projectId?.toString(),
+              alertId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+        }
+      })
       .catch((error: Error) => {
         logger.error(
           `Critical error in AlertService sequential operations: ${error}`,

package/Server/Services/IncidentService.ts

@@ -89,6 +89,7 @@ import LLMService, {
 } from "../Utils/LLM/LLMService";
 import LlmProviderService from "./LlmProviderService";
 import LlmProvider from "../../Models/DatabaseModels/LlmProvider";
+import SentinelIncidentInvestigationRunner from "../Utils/AI/Sentinel/IncidentInvestigationRunner";
 import IncidentAIContextBuilder, {
   AIGenerationContext,
   IncidentContextData,
@@ -1241,6 +1242,30 @@ export class Service extends DatabaseService<Model> {
           );
         }
       })
+      .then(async () => {
+        /*
+         * Sentinel (AI SRE): automatically investigate the new incident and post
+         * a cited root cause analysis to the incident timeline + Slack/Teams.
+         * Runs last so the workspace channels already exist, and is gated per
+         * project (opt-in) + requires a configured LLM provider. Read-only.
+         */
+        try {
+          if (createdItem.projectId && createdItem.id) {
+            await SentinelIncidentInvestigationRunner.investigateNewIncident({
+              incidentId: createdItem.id,
+              projectId: createdItem.projectId,
+            });
+          }
+        } catch (error) {
+          logger.error(
+            `Sentinel incident investigation failed in IncidentService.onCreateSuccess: ${error}`,
+            {
+              projectId: createdItem.projectId?.toString(),
+              incidentId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+        }
+      })
       .catch((error: Error) => {
         logger.error(
           `Critical error in IncidentService sequential operations: ${error}`,

package/Server/Services/IncidentStateTimelineService.ts

@@ -27,6 +27,7 @@ import IncidentRole from "../../Models/DatabaseModels/IncidentRole";
 import { IsBillingEnabled } from "../EnvironmentConfig";
 import logger, { LogAttributes } from "../Utils/Logger";
 import IncidentFeedService from "./IncidentFeedService";
+import SentinelIncidentPostmortemRunner from "../Utils/AI/Sentinel/IncidentPostmortemRunner";
 import { IncidentFeedEventType } from "../../Models/DatabaseModels/IncidentFeed";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
@@ -555,6 +556,24 @@ ${createdItem.rootCause}`,
           createdItem.startsAt || undefined,
         );
       }
+
+      /*
+       * Sentinel: auto-draft a postmortem now that the incident is resolved
+       * (gated per project; never overwrites an existing postmortem).
+       */
+      SentinelIncidentPostmortemRunner.draftPostmortemOnResolve({
+        incidentId: createdItem.incidentId!,
+        projectId: createdItem.projectId!,
+      }).catch((error: Error) => {
+        logger.error(`Sentinel auto-postmortem failed on resolve:`, {
+          projectId: createdItem.projectId?.toString(),
+          incidentId: createdItem.incidentId?.toString(),
+        } as LogAttributes);
+        logger.error(error, {
+          projectId: createdItem.projectId?.toString(),
+          incidentId: createdItem.incidentId?.toString(),
+        } as LogAttributes);
+      });
     }
 
     if (onCreate.carryForward.publicNote) {