@oneuptime/common 11.3.26 → 11.4.0

package/Server/API/AlertAPI.ts

@@ -122,7 +122,7 @@ export default class AlertAPI extends BaseAPI<Alert, AlertServiceType> {
       alertId: alertId,
       messages: aiContext.messages,
       maxTokens: 4096,
-      temperature: 0.7,
+      temperature: 0.2,
     };
 
     if (props.userId) {

package/Server/API/IncidentAPI.ts

@@ -230,7 +230,7 @@ export default class IncidentAPI extends BaseAPI<
       incidentId: incidentId,
       messages: aiContext.messages,
       maxTokens: 8192,
-      temperature: 0.7,
+      temperature: 0.2,
     };
 
     if (props.userId) {
@@ -347,7 +347,7 @@ export default class IncidentAPI extends BaseAPI<
       incidentId: incidentId,
       messages: aiContext.messages,
       maxTokens: 4096,
-      temperature: 0.7,
+      temperature: 0.2,
     };
 
     if (props.userId) {

package/Server/API/IncidentEpisodeAPI.ts

@@ -133,7 +133,7 @@ export default class IncidentEpisodeAPI extends BaseAPI<
       feature: "Incident Episode Postmortem",
       messages: aiContext.messages,
       maxTokens: 8192,
-      temperature: 0.7,
+      temperature: 0.2,
     };
 
     if (props.userId) {

package/Server/API/LlmProviderAPI.ts

@@ -42,6 +42,7 @@ export default class LlmProviderAPI extends BaseAPI<
               select: {
                 name: true,
                 description: true,
+                costPerMillionTokensInUSDCents: true,
               },
               props: {
                 isRoot: true,

package/Server/API/ScheduledMaintenanceAPI.ts

@@ -147,7 +147,7 @@ export default class ScheduledMaintenanceAPI extends BaseAPI<
       scheduledMaintenanceId: scheduledMaintenanceId,
       messages: aiContext.messages,
       maxTokens: 4096,
-      temperature: 0.7,
+      temperature: 0.2,
     };
 
     if (props.userId) {

package/Server/Infrastructure/Postgres/SchemaMigrations/1783470000000-AddIncomingCallPolicyRoutingPhoneNumberUnique.ts

@@ -0,0 +1,25 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddIncomingCallPolicyRoutingPhoneNumberUnique1783470000000
+  implements MigrationInterface
+{
+  public name = "AddIncomingCallPolicyRoutingPhoneNumberUnique1783470000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    /*
+     * Enforce that a routing phone number belongs to at most one policy. The
+     * inbound /voice webhook resolves the policy by routingPhoneNumber and
+     * assumes a single match. Partial index so policies without a number (NULL)
+     * are unconstrained and soft-deleted rows never collide.
+     */
+    await queryRunner.query(
+      `CREATE UNIQUE INDEX "IDX_INCOMING_CALL_POLICY_ROUTING_PHONE_UNIQUE" ON "IncomingCallPolicy" ("routingPhoneNumber") WHERE ("deletedAt" IS NULL AND "routingPhoneNumber" IS NOT NULL)`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_INCOMING_CALL_POLICY_ROUTING_PHONE_UNIQUE"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1783510935686-FixNotNullForeignKeysOnDelete.ts

@@ -0,0 +1,81 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/**
+ * These foreign keys previously used `ON DELETE SET NULL` even though their
+ * referencing columns are `NOT NULL`. Deleting the referenced row (e.g. a User
+ * or an IncidentRole) therefore tried to set the column to NULL and failed with
+ * a not-null constraint violation. Since each of these columns is required, the
+ * dependent row is meaningless without its parent, so the correct behavior is
+ * `ON DELETE CASCADE`.
+ */
+export class FixNotNullForeignKeysOnDelete1783510935686
+  implements MigrationInterface
+{
+  public name = "FixNotNullForeignKeysOnDelete1783510935686";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "IncidentMember" DROP CONSTRAINT "FK_6e8334ec5b25d596548c88d0832"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyTimeLog" DROP CONSTRAINT "FK_43da7ffeee531e9452d36a89ba5"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IncidentEpisodeRoleMember" DROP CONSTRAINT "FK_08c8933e37d14069c0d18b34f14"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyUserOverride" DROP CONSTRAINT "FK_4b3f696aaaf327b245ebeb3d146"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyUserOverride" DROP CONSTRAINT "FK_8771068ec4c16763a7ff796895d"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IncidentMember" ADD CONSTRAINT "FK_6e8334ec5b25d596548c88d0832" FOREIGN KEY ("incidentRoleId") REFERENCES "IncidentRole"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyTimeLog" ADD CONSTRAINT "FK_43da7ffeee531e9452d36a89ba5" FOREIGN KEY ("userId") REFERENCES "User"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IncidentEpisodeRoleMember" ADD CONSTRAINT "FK_08c8933e37d14069c0d18b34f14" FOREIGN KEY ("incidentRoleId") REFERENCES "IncidentRole"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyUserOverride" ADD CONSTRAINT "FK_4b3f696aaaf327b245ebeb3d146" FOREIGN KEY ("overrideUserId") REFERENCES "User"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyUserOverride" ADD CONSTRAINT "FK_8771068ec4c16763a7ff796895d" FOREIGN KEY ("routeAlertsToUserId") REFERENCES "User"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyUserOverride" DROP CONSTRAINT "FK_8771068ec4c16763a7ff796895d"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyUserOverride" DROP CONSTRAINT "FK_4b3f696aaaf327b245ebeb3d146"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IncidentEpisodeRoleMember" DROP CONSTRAINT "FK_08c8933e37d14069c0d18b34f14"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyTimeLog" DROP CONSTRAINT "FK_43da7ffeee531e9452d36a89ba5"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IncidentMember" DROP CONSTRAINT "FK_6e8334ec5b25d596548c88d0832"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyUserOverride" ADD CONSTRAINT "FK_8771068ec4c16763a7ff796895d" FOREIGN KEY ("routeAlertsToUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyUserOverride" ADD CONSTRAINT "FK_4b3f696aaaf327b245ebeb3d146" FOREIGN KEY ("overrideUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IncidentEpisodeRoleMember" ADD CONSTRAINT "FK_08c8933e37d14069c0d18b34f14" FOREIGN KEY ("incidentRoleId") REFERENCES "IncidentRole"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyTimeLog" ADD CONSTRAINT "FK_43da7ffeee531e9452d36a89ba5" FOREIGN KEY ("userId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IncidentMember" ADD CONSTRAINT "FK_6e8334ec5b25d596548c88d0832" FOREIGN KEY ("incidentRoleId") REFERENCES "IncidentRole"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1783515836148-MigrationName.ts

@@ -0,0 +1,25 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1783515836148 implements MigrationInterface {
+  public name = "MigrationName1783515836148";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    // HIGH-5: schedule timezone (nullable => existing schedules keep legacy server-local behavior).
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicySchedule" ADD "timezone" character varying(100)`,
+    );
+    // M-5: bounded retry counter for escalation rules that momentarily resolve to no on-call user.
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyExecutionLog" ADD "scheduleGapRetryCount" integer NOT NULL DEFAULT '0'`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyExecutionLog" DROP COLUMN "scheduleGapRetryCount"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicySchedule" DROP COLUMN "timezone"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1783523076215-AddMigrationFailureTable.ts

@@ -0,0 +1,35 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddMigrationFailureTable1783523076215
+  implements MigrationInterface
+{
+  public name = "AddMigrationFailureTable1783523076215";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "MigrationFailure" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "migrationName" character varying(100) NOT NULL, "migrationType" character varying(100) NOT NULL, "errorMessage" text NOT NULL, "errorStack" text, "attemptedAt" TIMESTAMP WITH TIME ZONE NOT NULL, "hostName" character varying(100), "appVersion" character varying(100), CONSTRAINT "PK_db298508bb9e7064755c660eb8b" PRIMARY KEY ("_id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_afd3cf5c9703ef3cd3f74bcecf" ON "MigrationFailure" ("migrationName") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_0e4fbfab7ec24121ac4fc66d64" ON "MigrationFailure" ("migrationType") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_d0f64f28973facc22ab17764e4" ON "MigrationFailure" ("attemptedAt") `,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_d0f64f28973facc22ab17764e4"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_0e4fbfab7ec24121ac4fc66d64"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_afd3cf5c9703ef3cd3f74bcecf"`,
+    );
+    await queryRunner.query(`DROP TABLE "MigrationFailure"`);
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1783598145111-AddEnableAutomaticIncidentInvestigationToProject.ts

@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddEnableAutomaticIncidentInvestigationToProject1783598145111
+  implements MigrationInterface
+{
+  public name = "AddEnableAutomaticIncidentInvestigationToProject1783598145111";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableAutomaticIncidentInvestigation" boolean NOT NULL DEFAULT false`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableAutomaticIncidentInvestigation"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1783615884106-AddInvestigationSubjectToAIRun.ts

@@ -0,0 +1,33 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddInvestigationSubjectToAIRun1783615884106
+  implements MigrationInterface
+{
+  public name = "AddInvestigationSubjectToAIRun1783615884106";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" ADD "triggeredByIncidentId" uuid`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" ADD "triggeredByAlertId" uuid`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_AIRun_triggeredByIncidentId" ON "AIRun" ("triggeredByIncidentId")`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_AIRun_triggeredByAlertId" ON "AIRun" ("triggeredByAlertId")`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "IDX_AIRun_triggeredByAlertId"`);
+    await queryRunner.query(`DROP INDEX "IDX_AIRun_triggeredByIncidentId"`);
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" DROP COLUMN "triggeredByAlertId"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" DROP COLUMN "triggeredByIncidentId"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1783632447975-AddEnableAutomaticAlertInvestigationToProject.ts

@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddEnableAutomaticAlertInvestigationToProject1783632447975
+  implements MigrationInterface
+{
+  public name = "AddEnableAutomaticAlertInvestigationToProject1783632447975";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableAutomaticAlertInvestigation" boolean NOT NULL DEFAULT false`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableAutomaticAlertInvestigation"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -413,6 +413,13 @@ import { AddAIChatModels1783363279075 } from "./1783363279075-AddAIChatModels";
 import { AddLlmProviderToAIConversation1783443471795 } from "./1783443471795-AddLlmProviderToAIConversation";
 import { AddAIChatWriteActionsAndWidgets1783453297388 } from "./1783453297388-AddAIChatWriteActionsAndWidgets";
 import { MigrationName1783461767405 } from "./1783461767405-MigrationName";
+import { AddIncomingCallPolicyRoutingPhoneNumberUnique1783470000000 } from "./1783470000000-AddIncomingCallPolicyRoutingPhoneNumberUnique";
+import { FixNotNullForeignKeysOnDelete1783510935686 } from "./1783510935686-FixNotNullForeignKeysOnDelete";
+import { MigrationName1783515836148 } from "./1783515836148-MigrationName";
+import { AddMigrationFailureTable1783523076215 } from "./1783523076215-AddMigrationFailureTable";
+import { AddEnableAutomaticIncidentInvestigationToProject1783598145111 } from "./1783598145111-AddEnableAutomaticIncidentInvestigationToProject";
+import { AddInvestigationSubjectToAIRun1783615884106 } from "./1783615884106-AddInvestigationSubjectToAIRun";
+import { AddEnableAutomaticAlertInvestigationToProject1783632447975 } from "./1783632447975-AddEnableAutomaticAlertInvestigationToProject";
 
 export default [
   InitialMigration,
@@ -830,4 +837,11 @@ export default [
   AddLlmProviderToAIConversation1783443471795,
   AddAIChatWriteActionsAndWidgets1783453297388,
   MigrationName1783461767405,
+  AddIncomingCallPolicyRoutingPhoneNumberUnique1783470000000,
+  FixNotNullForeignKeysOnDelete1783510935686,
+  MigrationName1783515836148,
+  AddMigrationFailureTable1783523076215,
+  AddEnableAutomaticIncidentInvestigationToProject1783598145111,
+  AddInvestigationSubjectToAIRun1783615884106,
+  AddEnableAutomaticAlertInvestigationToProject1783632447975,
 ];

package/Server/Infrastructure/PostgresDatabase.ts

@@ -1,5 +1,6 @@
 import logger from "../Utils/Logger";
 import DatabaseDataSourceOptions from "./Postgres/DataSourceOptions";
+import { recordSchemaMigrationFailureBestEffort } from "../Utils/Database/MigrationFailureLog";
 import Sleep from "../../Types/Sleep";
 import { DataSource, DataSourceOptions, QueryRunner } from "typeorm";
 import { createDatabase, dropDatabase } from "typeorm-extension";
@@ -95,6 +96,16 @@ export default class Database {
     } catch (err) {
       logger.error("Postgres Database Connection Failed");
       logger.error(err);
+
+      /*
+       * When this process runs schema migrations on boot (migrationsRun=true),
+       * connect() also fails if a migration threw. Record which migration
+       * failed and why — on a throwaway connection, since the DataSource above
+       * is unusable — so the admin health page can explain the pending schema.
+       * Best-effort and self-contained: it never throws and never masks `err`.
+       */
+      await recordSchemaMigrationFailureBestEffort(dataSourceOptions, err);
+
       throw err;
     }
   }

package/Server/Services/AIService.ts

@@ -11,7 +11,10 @@ import LLMService, {
   LLMMessage,
   LLMToolCall,
   LLMToolDefinition,
+  LLMUsage,
 } from "../Utils/LLM/LLMService";
+import LlmType from "../../Types/LLM/LlmType";
+import { Span, trace } from "@opentelemetry/api";
 import LlmProvider from "../../Models/DatabaseModels/LlmProvider";
 import LlmLog from "../../Models/DatabaseModels/LlmLog";
 import LlmLogStatus from "../../Types/LlmLogStatus";
@@ -129,9 +132,17 @@ export class Service extends BaseService {
       logEntry.aiRunId = request.aiRunId;
     }
 
-    // Check if billing should apply
+    /*
+     * Check if billing should apply. Only bill for the global (OneUptime-hosted)
+     * provider, and only when it actually has a per-token cost. A free global
+     * provider (costPerMillionTokensInUSDCents = 0, the default) consumes no
+     * balance, so it must not require or block on one either — otherwise a $0
+     * provider would still fail with "Insufficient AI balance".
+     */
     const shouldBill: boolean =
-      IsBillingEnabled && (llmProvider.isGlobalLlm || false);
+      IsBillingEnabled &&
+      (llmProvider.isGlobalLlm || false) &&
+      (llmProvider.costPerMillionTokensInUSDCents || 0) > 0;
 
     // Check balance if billing enabled and using global provider
     if (shouldBill) {
@@ -234,6 +245,18 @@ export class Service extends BaseService {
         }
       }
 
+      /*
+       * Emit gen_ai.* semantic-convention attributes on the active span so
+       * OneUptime's own AI usage is a first-class LLM span in OneUptime's own
+       * telemetry (dogfooding — LlmSpanUtil detects these). Never fails the call.
+       */
+      this.setGenAiSpanAttributes({
+        llmType: llmProvider.llmType,
+        modelName: llmConfig.modelName,
+        usage: response.usage,
+        costInUSDCents: logEntry.costInUSDCents,
+      });
+
       // Save log entry
       const savedLog: LlmLog = await LlmLogService.create({
         data: logEntry,
@@ -261,6 +284,55 @@ export class Service extends BaseService {
       throw error;
     }
   }
+
+  /*
+   * Set gen_ai.* attributes (OpenTelemetry GenAI semantic conventions) on the
+   * currently-active span. The @CaptureSpan()-wrapped caller owns that span, so
+   * LlmSpanUtil recognizes these calls as first-class LLM spans.
+   */
+  private setGenAiSpanAttributes(data: {
+    llmType: LlmType;
+    modelName?: string | undefined;
+    usage?: LLMUsage | undefined;
+    costInUSDCents?: number | undefined;
+  }): void {
+    try {
+      const span: Span | undefined = trace.getActiveSpan();
+      if (!span) {
+        return;
+      }
+
+      span.setAttribute("gen_ai.system", data.llmType.toString());
+      span.setAttribute("gen_ai.provider.name", data.llmType.toString());
+      span.setAttribute("gen_ai.operation.name", "chat");
+
+      if (data.modelName) {
+        span.setAttribute("gen_ai.request.model", data.modelName);
+        span.setAttribute("gen_ai.response.model", data.modelName);
+      }
+
+      if (data.usage) {
+        span.setAttribute(
+          "gen_ai.usage.input_tokens",
+          data.usage.promptTokens || 0,
+        );
+        span.setAttribute(
+          "gen_ai.usage.output_tokens",
+          data.usage.completionTokens || 0,
+        );
+        span.setAttribute(
+          "gen_ai.usage.total_tokens",
+          data.usage.totalTokens || 0,
+        );
+      }
+
+      if (data.costInUSDCents) {
+        span.setAttribute("gen_ai.usage.cost_usd", data.costInUSDCents / 100);
+      }
+    } catch {
+      // Telemetry must never fail the LLM call.
+    }
+  }
 }
 
 export default new Service();

package/Server/Services/AlertService.ts

@@ -67,6 +67,7 @@ import AlertLabelRuleEngineService from "./AlertLabelRuleEngineService";
 import AlertOnCallRuleEngineService from "./AlertOnCallRuleEngineService";
 import AlertOwnerRuleEngineService from "./AlertOwnerRuleEngineService";
 import RunbookRuleEngineService from "./RunbookRuleEngineService";
+import SentinelAlertInvestigationRunner from "../Utils/AI/Sentinel/AlertInvestigationRunner";
 import AlertPrivacyRuleEngineService from "./AlertPrivacyRuleEngineService";
 import ProjectService from "./ProjectService";
 
@@ -558,6 +559,30 @@ export class Service extends DatabaseService<Model> {
           );
         }
       })
+      .then(async () => {
+        /*
+         * Sentinel (AI SRE): automatically investigate the new alert and post a
+         * cited root cause analysis to the alert timeline + Slack/Teams. Runs
+         * last, is gated per project (opt-in) + requires an LLM provider, and is
+         * read-only.
+         */
+        try {
+          if (createdItem.projectId && createdItem.id) {
+            await SentinelAlertInvestigationRunner.investigateNewAlert({
+              alertId: createdItem.id,
+              projectId: createdItem.projectId,
+            });
+          }
+        } catch (error) {
+          logger.error(
+            `Sentinel alert investigation failed in AlertService.onCreateSuccess: ${error}`,
+            {
+              projectId: createdItem.projectId?.toString(),
+              alertId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+        }
+      })
       .catch((error: Error) => {
         logger.error(
           `Critical error in AlertService sequential operations: ${error}`,

package/Server/Services/AnalyticsDatabaseService.ts

@@ -1899,7 +1899,16 @@ export default class AnalyticsDatabaseService<
     return data;
   }
 
-  protected async getException(error: Exception): Promise<void> {
+  /*
+   * Rethrow hook for the catch blocks below. MUST stay synchronous and
+   * `never`-returning: the call sites use `throw this.getException(error)`,
+   * so if this were `async` it would return a Promise that `throw` then
+   * throws verbatim (never awaited) — the real exception is lost, callers up
+   * the stack catch a bare Promise (surfacing as "[object Promise]"), and the
+   * un-awaited rejection becomes an unhandled rejection. Throwing directly
+   * propagates the original exception on the synchronous throw path.
+   */
+  protected getException(error: Exception): never {
     throw error;
   }
 

package/Server/Services/DatabaseService.ts

@@ -443,7 +443,16 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
     return Promise.resolve(error);
   }
 
-  protected async getException(error: Exception): Promise<void> {
+  /*
+   * Rethrow hook for the catch blocks below. MUST stay synchronous and
+   * `never`-returning: the call sites use `throw this.getException(error)`,
+   * so if this were `async` it would return a Promise that `throw` then
+   * throws verbatim (never awaited) — the real exception is lost, callers up
+   * the stack catch a bare Promise (surfacing as "[object Promise]"), and the
+   * un-awaited rejection becomes an unhandled rejection. Throwing directly
+   * propagates the original exception on the synchronous throw path.
+   */
+  protected getException(error: Exception): never {
     throw error;
   }
 

package/Server/Services/IncidentService.ts

@@ -89,6 +89,7 @@ import LLMService, {
 } from "../Utils/LLM/LLMService";
 import LlmProviderService from "./LlmProviderService";
 import LlmProvider from "../../Models/DatabaseModels/LlmProvider";
+import SentinelIncidentInvestigationRunner from "../Utils/AI/Sentinel/IncidentInvestigationRunner";
 import IncidentAIContextBuilder, {
   AIGenerationContext,
   IncidentContextData,
@@ -1241,6 +1242,30 @@ export class Service extends DatabaseService<Model> {
           );
         }
       })
+      .then(async () => {
+        /*
+         * Sentinel (AI SRE): automatically investigate the new incident and post
+         * a cited root cause analysis to the incident timeline + Slack/Teams.
+         * Runs last so the workspace channels already exist, and is gated per
+         * project (opt-in) + requires a configured LLM provider. Read-only.
+         */
+        try {
+          if (createdItem.projectId && createdItem.id) {
+            await SentinelIncidentInvestigationRunner.investigateNewIncident({
+              incidentId: createdItem.id,
+              projectId: createdItem.projectId,
+            });
+          }
+        } catch (error) {
+          logger.error(
+            `Sentinel incident investigation failed in IncidentService.onCreateSuccess: ${error}`,
+            {
+              projectId: createdItem.projectId?.toString(),
+              incidentId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+        }
+      })
       .catch((error: Error) => {
         logger.error(
           `Critical error in IncidentService sequential operations: ${error}`,
@@ -3479,7 +3504,7 @@ ${incidentSeverity.name}
     const response: LLMCompletionResponse = await LLMService.getCompletion({
       llmProviderConfig: llmConfig,
       messages: aiContext.messages,
-      temperature: 0.7,
+      temperature: 0.2,
     });
 
     return response.content;

package/Server/Services/IncidentStateTimelineService.ts

@@ -27,6 +27,7 @@ import IncidentRole from "../../Models/DatabaseModels/IncidentRole";
 import { IsBillingEnabled } from "../EnvironmentConfig";
 import logger, { LogAttributes } from "../Utils/Logger";
 import IncidentFeedService from "./IncidentFeedService";
+import SentinelIncidentPostmortemRunner from "../Utils/AI/Sentinel/IncidentPostmortemRunner";
 import { IncidentFeedEventType } from "../../Models/DatabaseModels/IncidentFeed";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
@@ -555,6 +556,24 @@ ${createdItem.rootCause}`,
           createdItem.startsAt || undefined,
         );
       }
+
+      /*
+       * Sentinel: auto-draft a postmortem now that the incident is resolved
+       * (gated per project; never overwrites an existing postmortem).
+       */
+      SentinelIncidentPostmortemRunner.draftPostmortemOnResolve({
+        incidentId: createdItem.incidentId!,
+        projectId: createdItem.projectId!,
+      }).catch((error: Error) => {
+        logger.error(`Sentinel auto-postmortem failed on resolve:`, {
+          projectId: createdItem.projectId?.toString(),
+          incidentId: createdItem.incidentId?.toString(),
+        } as LogAttributes);
+        logger.error(error, {
+          projectId: createdItem.projectId?.toString(),
+          incidentId: createdItem.incidentId?.toString(),
+        } as LogAttributes);
+      });
     }
 
     if (onCreate.carryForward.publicNote) {