@oneuptime/common 11.5.4 → 11.5.6

package/Server/Services/IncidentService.ts

@@ -6,6 +6,9 @@ import FindBy from "../Types/Database/FindBy";
 import { OnCreate, OnDelete, OnFind, OnUpdate } from "../Types/Database/Hooks";
 import QueryHelper from "../Types/Database/QueryHelper";
 import DatabaseService from "./DatabaseService";
+import AIRunService from "./AIRunService";
+import AIRunType from "../../Types/AI/AIRunType";
+import AIRunStatus from "../../Types/AI/AIRunStatus";
 import IncidentOwnerTeamService from "./IncidentOwnerTeamService";
 import IncidentOwnerUserService from "./IncidentOwnerUserService";
 import IncidentStateService from "./IncidentStateService";
@@ -84,7 +87,7 @@ import ProjectService from "./ProjectService";
 import IncidentTemplateService from "./IncidentTemplateService";
 import IncidentTemplate from "../../Models/DatabaseModels/IncidentTemplate";
 import AIService, { AILogResponse } from "./AIService";
-import SentinelIncidentInvestigationRunner from "../Utils/AI/Sentinel/IncidentInvestigationRunner";
+import AIIncidentInvestigationRunner from "../Utils/AI/SRE/IncidentInvestigationRunner";
 import IncidentAIContextBuilder, {
   AIGenerationContext,
   IncidentContextData,
@@ -1239,21 +1242,21 @@ export class Service extends DatabaseService<Model> {
       })
       .then(async () => {
         /*
-         * Sentinel (AI SRE): automatically investigate the new incident and post
+         * AI (AI SRE): automatically investigate the new incident and post
          * a cited root cause analysis to the incident timeline + Slack/Teams.
          * Runs last so the workspace channels already exist, and is gated per
          * project (opt-in) + requires a configured LLM provider. Read-only.
          */
         try {
           if (createdItem.projectId && createdItem.id) {
-            await SentinelIncidentInvestigationRunner.investigateNewIncident({
+            await AIIncidentInvestigationRunner.investigateNewIncident({
               incidentId: createdItem.id,
               projectId: createdItem.projectId,
             });
           }
         } catch (error) {
           logger.error(
-            `Sentinel incident investigation failed in IncidentService.onCreateSuccess: ${error}`,
+            `AI incident investigation failed in IncidentService.onCreateSuccess: ${error}`,
             {
               projectId: createdItem.projectId?.toString(),
               incidentId: createdItem.id?.toString(),
@@ -2795,14 +2798,23 @@ ${incidentSeverity.name}
     return Service.DEFAULT_METRIC_RETENTION_DAYS;
   }
 
+  /*
+   * Load the incident + its owners and build the attribute set shared by
+   * every incident metric (all values strings for ClickHouse
+   * Map(String, String) storage; arrays joined comma-separated). Factored
+   * out of refreshIncidentMetrics so one-off metric writers — e.g. the
+   * AI time-to-rca metric recorded when an investigation posts its
+   * analysis — record the exact same attribute shape.
+   */
   @CaptureSpan()
-  public async refreshIncidentMetrics(data: {
+  public async getIncidentMetricContext(data: {
     incidentId: ObjectID;
-  }): Promise<void> {
+  }): Promise<{ incident: Model; baseMetricAttributes: JSONObject }> {
     const incident: Model | null = await this.findOneById({
       id: data.incidentId,
       select: {
         projectId: true,
+        createdAt: true,
         declaredAt: true,
         postmortemPostedAt: true,
         monitors: {
@@ -2898,6 +2910,161 @@ ${incidentSeverity.name}
         return Boolean(name);
       }) as Array<string>;
 
+    /*
+     * common attributes shared by all incident metrics
+     * All values must be strings for ClickHouse Map(String, String) storage.
+     * Arrays are joined as comma-separated strings.
+     */
+    const baseMetricAttributes: JSONObject = {
+      incidentId: data.incidentId.toString(),
+      projectId: incident.projectId.toString(),
+      monitorIds: (
+        incident.monitors
+          ?.map((monitor: Monitor) => {
+            return monitor._id?.toString();
+          })
+          .filter(Boolean) || []
+      ).join(", "),
+      monitorNames: (
+        incident.monitors
+          ?.map((monitor: Monitor) => {
+            return monitor.name?.toString();
+          })
+          .filter(Boolean) || []
+      ).join(", "),
+      incidentSeverityId: incident.incidentSeverity?._id?.toString(),
+      incidentSeverityName: incident.incidentSeverity?.name?.toString(),
+      ownerUserIds: ownerUserIds.join(", "),
+      ownerUserNames: ownerUserNames.join(", "),
+      ownerTeamIds: ownerTeamIds.join(", "),
+      ownerTeamNames: ownerTeamNames.join(", "),
+    };
+
+    return { incident, baseMetricAttributes };
+  }
+
+  /*
+   * AI measurement layer: record how long the incident waited for its
+   * AI root-cause analysis — seconds from incident creation to now, written
+   * once from IncidentInvestigationRunner.postAnalysis. Deliberately NOT
+   * part of refreshIncidentMetrics: the refresh's replace-list excludes
+   * this metric name, so refreshes never tombstone it.
+   */
+  @CaptureSpan()
+  public async recordTimeToRootCausePostedMetric(data: {
+    incidentId: ObjectID;
+  }): Promise<void> {
+    const {
+      incident,
+      baseMetricAttributes,
+    }: { incident: Model; baseMetricAttributes: JSONObject } =
+      await this.getIncidentMetricContext({ incidentId: data.incidentId });
+
+    const now: Date = OneUptimeDate.getCurrentDate();
+    const incidentCreatedAt: Date =
+      incident.createdAt || incident.declaredAt || now;
+
+    const metricRetentionDays: number = await this.getMetricRetentionDays();
+    const retentionDate: Date = OneUptimeDate.addRemoveDays(
+      OneUptimeDate.getCurrentDate(),
+      metricRetentionDays,
+    );
+
+    const timeToRcaMetric: MutableMetric = new MutableMetric();
+
+    timeToRcaMetric.projectId = incident.projectId!;
+    timeToRcaMetric.primaryEntityId = incident.id!;
+    timeToRcaMetric.primaryEntityType = ServiceType.Incident;
+    timeToRcaMetric.name = IncidentMetricType.TimeToRootCausePosted;
+    timeToRcaMetric.metricPointId = IncidentMetricType.TimeToRootCausePosted;
+    timeToRcaMetric.value = OneUptimeDate.getDifferenceInSeconds(
+      now,
+      incidentCreatedAt,
+    );
+    timeToRcaMetric.attributes = {
+      ...baseMetricAttributes,
+      // Every time-to-rca point is AI-posted by construction.
+      aiInvestigated: "true",
+    };
+    timeToRcaMetric.attributeKeys = TelemetryUtil.getAttributeKeys(
+      timeToRcaMetric.attributes,
+    );
+    timeToRcaMetric.time = now;
+    timeToRcaMetric.timeUnixNano = OneUptimeDate.toUnixNano(now);
+    timeToRcaMetric.metricPointType = MetricPointType.Sum;
+    timeToRcaMetric.retentionDate = retentionDate;
+
+    await MutableMetricService.createMutableMetrics({
+      metrics: [timeToRcaMetric],
+    });
+
+    // Register the metric type so it shows up in the type catalog.
+    const metricType: MetricType = new MetricType();
+    metricType.name = IncidentMetricType.TimeToRootCausePosted;
+    metricType.description =
+      "Time from incident creation to the AI investigation's posted root-cause analysis";
+    metricType.unit = "seconds";
+
+    TelemetryUtil.indexMetricNameServiceNameMap({
+      metricNameServiceNameMap: {
+        [IncidentMetricType.TimeToRootCausePosted]: metricType,
+      },
+      projectId: incident.projectId!,
+    }).catch((err: Error) => {
+      logger.error(err, {
+        projectId: incident.projectId?.toString(),
+        incidentId: incident.id?.toString(),
+      } as LogAttributes);
+    });
+  }
+
+  @CaptureSpan()
+  public async refreshIncidentMetrics(data: {
+    incidentId: ObjectID;
+  }): Promise<void> {
+    const {
+      incident,
+      baseMetricAttributes,
+    }: { incident: Model; baseMetricAttributes: JSONObject } =
+      await this.getIncidentMetricContext({ incidentId: data.incidentId });
+
+    // getIncidentMetricContext guarantees this; re-checked for TS narrowing.
+    if (!incident.projectId) {
+      throw new BadDataException("Incident Project ID not found");
+    }
+
+    /*
+     * aiInvestigated dimension for MTTA/MTTR (AI measurement layer):
+     * did a completed AI investigation run for this incident? One
+     * indexed countBy per metric refresh (triggeredByIncidentId is indexed)
+     * — acceptable at refresh frequency. Failure must never break metric
+     * recording, so this is best-effort false.
+     */
+    let aiInvestigated: boolean = false;
+    try {
+      const completedInvestigationCount: PositiveNumber =
+        await AIRunService.countBy({
+          query: {
+            runType: AIRunType.Investigation,
+            status: AIRunStatus.Completed,
+            triggeredByIncidentId: data.incidentId,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+
+      aiInvestigated = completedInvestigationCount.toNumber() > 0;
+    } catch (err) {
+      logger.error(
+        err as Error,
+        {
+          projectId: incident.projectId?.toString(),
+          incidentId: incident.id?.toString(),
+        } as LogAttributes,
+      );
+    }
+
     // get incident state timeline
 
     const incidentStateTimelines: Array<IncidentStateTimeline> =
@@ -2973,36 +3140,6 @@ ${incidentSeverity.name}
 
       const metricTypesMap: Dictionary<MetricType> = {};
 
-      /*
-       * common attributes shared by all incident metrics
-       * All values must be strings for ClickHouse Map(String, String) storage.
-       * Arrays are joined as comma-separated strings.
-       */
-      const baseMetricAttributes: JSONObject = {
-        incidentId: data.incidentId.toString(),
-        projectId: incident.projectId.toString(),
-        monitorIds: (
-          incident.monitors
-            ?.map((monitor: Monitor) => {
-              return monitor._id?.toString();
-            })
-            .filter(Boolean) || []
-        ).join(", "),
-        monitorNames: (
-          incident.monitors
-            ?.map((monitor: Monitor) => {
-              return monitor.name?.toString();
-            })
-            .filter(Boolean) || []
-        ).join(", "),
-        incidentSeverityId: incident.incidentSeverity?._id?.toString(),
-        incidentSeverityName: incident.incidentSeverity?.name?.toString(),
-        ownerUserIds: ownerUserIds.join(", "),
-        ownerUserNames: ownerUserNames.join(", "),
-        ownerTeamIds: ownerTeamIds.join(", "),
-        ownerTeamNames: ownerTeamNames.join(", "),
-      };
-
       const incidentCountMetric: MutableMetric = new MutableMetric();
 
       incidentCountMetric.projectId = incident.projectId;
@@ -3068,7 +3205,11 @@ ${incidentSeverity.name}
               OneUptimeDate.getCurrentDate(),
             incidentStartsAt,
           );
-          timeToAcknowledgeMetric.attributes = { ...baseMetricAttributes };
+          // aiInvestigated: the MTTA with/without-AI dimension.
+          timeToAcknowledgeMetric.attributes = {
+            ...baseMetricAttributes,
+            aiInvestigated: aiInvestigated.toString(),
+          };
           timeToAcknowledgeMetric.attributeKeys =
             TelemetryUtil.getAttributeKeys(timeToAcknowledgeMetric.attributes);
 
@@ -3119,7 +3260,11 @@ ${incidentSeverity.name}
             OneUptimeDate.getCurrentDate(),
           incidentStartsAt,
         );
-        timeToResolveMetric.attributes = { ...baseMetricAttributes };
+        // aiInvestigated: the MTTR with/without-AI dimension.
+        timeToResolveMetric.attributes = {
+          ...baseMetricAttributes,
+          aiInvestigated: aiInvestigated.toString(),
+        };
         timeToResolveMetric.attributeKeys = TelemetryUtil.getAttributeKeys(
           timeToResolveMetric.attributes,
         );

package/Server/Services/IncidentStateTimelineService.ts

@@ -27,7 +27,8 @@ import IncidentRole from "../../Models/DatabaseModels/IncidentRole";
 import { IsBillingEnabled } from "../EnvironmentConfig";
 import logger, { LogAttributes } from "../Utils/Logger";
 import IncidentFeedService from "./IncidentFeedService";
-import SentinelIncidentPostmortemRunner from "../Utils/AI/Sentinel/IncidentPostmortemRunner";
+import AIIncidentPostmortemRunner from "../Utils/AI/SRE/IncidentPostmortemRunner";
+import InvestigationGrader from "../Utils/AI/SRE/InvestigationGrader";
 import { IncidentFeedEventType } from "../../Models/DatabaseModels/IncidentFeed";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
@@ -558,14 +559,33 @@ ${createdItem.rootCause}`,
       }
 
       /*
-       * Sentinel: auto-draft a postmortem now that the incident is resolved
+       * AI: auto-draft a postmortem now that the incident is resolved
        * (gated per project; never overwrites an existing postmortem).
        */
-      SentinelIncidentPostmortemRunner.draftPostmortemOnResolve({
+      AIIncidentPostmortemRunner.draftPostmortemOnResolve({
         incidentId: createdItem.incidentId!,
         projectId: createdItem.projectId!,
       }).catch((error: Error) => {
-        logger.error(`Sentinel auto-postmortem failed on resolve:`, {
+        logger.error(`AI auto-postmortem failed on resolve:`, {
+          projectId: createdItem.projectId?.toString(),
+          incidentId: createdItem.incidentId?.toString(),
+        } as LogAttributes);
+        logger.error(error, {
+          projectId: createdItem.projectId?.toString(),
+          incidentId: createdItem.incidentId?.toString(),
+        } as LogAttributes);
+      });
+
+      /*
+       * AI measurement layer: grade the completed investigation (if
+       * any) against the human-recorded root cause. Fire-and-forget like
+       * the postmortem draft above — must never block or fail the resolve.
+       */
+      InvestigationGrader.gradeInvestigationOnResolve({
+        incidentId: createdItem.incidentId!,
+        projectId: createdItem.projectId!,
+      }).catch((error: Error) => {
+        logger.error(`AI investigation grading failed on resolve:`, {
           projectId: createdItem.projectId?.toString(),
           incidentId: createdItem.incidentId?.toString(),
         } as LogAttributes);

package/Server/Services/Index.ts

@@ -13,6 +13,7 @@ import CallLogService from "./CallLogService";
 import CallService from "./CallService";
 import DataMigrationService from "./DataMigrationService";
 import MigrationFailureService from "./MigrationFailureService";
+import InstanceHealthLogService from "./InstanceHealthLogService";
 import DomainService from "./DomainService";
 import EmailLogService from "./EmailLogService";
 import EmailVerificationTokenService from "./EmailVerificationTokenService";
@@ -81,6 +82,7 @@ import MonitorOwnerUserService from "./MonitorOwnerUserService";
 import MonitorProbeService from "./MonitorProbeService";
 import MonitorSecretService from "./MonitorSecretService";
 import RunbookSecretService from "./RunbookSecretService";
+import AIInsightService from "./AIInsightService";
 
 // Monitors
 import MonitorService from "./MonitorService";
@@ -104,7 +106,6 @@ import ProbeService from "./ProbeService";
 import AIAgentService from "./AIAgentService";
 import AIAgentOwnerUserService from "./AIAgentOwnerUserService";
 import AIAgentOwnerTeamService from "./AIAgentOwnerTeamService";
-import AIAgentTaskLogService from "./AIAgentTaskLogService";
 import AIAgentTaskPullRequestService from "./AIAgentTaskPullRequestService";
 import ProjectCallSMSConfigService from "./ProjectCallSMSConfigService";
 import ProjectService from "./ProjectService";
@@ -137,7 +138,6 @@ import ScheduledMaintenanceStateTimelineService from "./ScheduledMaintenanceStat
 import ServiceOwnerTeamService from "./ServiceOwnerTeamService";
 import ServiceOwnerUserService from "./ServiceOwnerUserService";
 import ServiceService from "./ServiceService";
-import ServiceCodeRepositoryService from "./ServiceCodeRepositoryService";
 import ShortLinkService from "./ShortLinkService";
 // SMS Log Service
 import SmsLogService from "./SmsLogService";
@@ -282,6 +282,7 @@ const services: Array<BaseService> = [
 
   DataMigrationService,
   MigrationFailureService,
+  InstanceHealthLogService,
   DomainService,
 
   EmailLogService,
@@ -346,6 +347,7 @@ const services: Array<BaseService> = [
   MonitorStatusTimelineService,
   MonitorSecretService,
   RunbookSecretService,
+  AIInsightService,
   MonitorFeedService,
 
   NotificationService,
@@ -365,7 +367,6 @@ const services: Array<BaseService> = [
   AIAgentService,
   AIAgentOwnerUserService,
   AIAgentOwnerTeamService,
-  AIAgentTaskLogService,
   AIAgentTaskPullRequestService,
   ProjectSsoService,
   ProjectOidcService,
@@ -456,7 +457,6 @@ const services: Array<BaseService> = [
   ServiceService,
   ServiceOwnerTeamService,
   ServiceOwnerUserService,
-  ServiceCodeRepositoryService,
 
   TelemetryExceptionService,
   TelemetryEntityService,

package/Server/Services/{AIAgentTaskLogService.ts → InstanceHealthLogService.ts}

@@ -1,5 +1,5 @@
 import DatabaseService from "./DatabaseService";
-import Model from "../../Models/DatabaseModels/AIAgentTaskLog";
+import Model from "../../Models/DatabaseModels/InstanceHealthLog";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {

package/Server/Services/LlmLogService.ts

@@ -35,6 +35,33 @@ export class Service extends DatabaseService<Model> {
 
     return Number(rows[0]?.total || 0);
   }
+
+  /*
+   * Per-run LLM usage for the server-mediated code-fix agent loop budgets
+   * (B4 Tier 0): how many completion calls a run has made and how many
+   * output tokens they produced. One SQL aggregate over the run's LlmLog
+   * rows — executeWithLogging writes exactly one row per call (success,
+   * error, or budget rejection), so the count is the honest call count and
+   * errored calls conservatively consume a slot. Runs are bounded to ~30
+   * minutes, far inside the 3-day cloud LlmLog retention.
+   */
+  @CaptureSpan()
+  public async getRunCompletionUsage(data: {
+    aiRunId: ObjectID;
+  }): Promise<{ completionCalls: number; outputTokens: number }> {
+    const rows: Array<{
+      calls: string | number | null;
+      outputTokens: string | number | null;
+    }> = await this.getRepository().manager.query(
+      `SELECT COUNT(*) AS "calls", COALESCE(SUM("completionTokens"), 0) AS "outputTokens" FROM "LlmLog" WHERE "aiRunId" = $1 AND "deletedAt" IS NULL`,
+      [data.aiRunId.toString()],
+    );
+
+    return {
+      completionCalls: Number(rows[0]?.calls || 0),
+      outputTokens: Number(rows[0]?.outputTokens || 0),
+    };
+  }
 }
 
 export default new Service();

package/Server/Services/LlmProviderService.ts

@@ -157,6 +157,114 @@ export class Service extends DatabaseService<Model> {
     return null;
   }
 
+  /*
+   * Resolve a provider the project OWNS — never the global fallback.
+   * Callers that may also use the shared global provider layer that
+   * fallback themselves (see getLlmProviderForMeteredAgentPath). Prefers
+   * the project default, else any project-owned provider.
+   */
+  @CaptureSpan()
+  public async getProjectOwnedLlmProvider(
+    projectId: ObjectID,
+  ): Promise<Model | null> {
+    const select: {
+      _id: boolean;
+      name: boolean;
+      llmType: boolean;
+      apiKey: boolean;
+      baseUrl: boolean;
+      modelName: boolean;
+      additionalParams: boolean;
+      isGlobalLlm: boolean;
+      costPerMillionTokensInUSDCents: boolean;
+    } = {
+      _id: true,
+      name: true,
+      llmType: true,
+      apiKey: true,
+      baseUrl: true,
+      modelName: true,
+      additionalParams: true,
+      isGlobalLlm: true,
+      costPerMillionTokensInUSDCents: true,
+    };
+
+    const defaultProvider: Model | null = await this.findOneBy({
+      query: {
+        projectId: projectId,
+        isDefault: true,
+      },
+      select,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (defaultProvider) {
+      return defaultProvider;
+    }
+
+    return this.findOneBy({
+      query: {
+        projectId: projectId,
+      },
+      sort: {
+        createdAt: SortOrder.Ascending,
+      },
+      select,
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+
+  /*
+   * Resolve the provider for the METERED agent path (B4 Tier 0): the
+   * server-mediated /ai-agent-data/llm-completion endpoint, whose calls run
+   * through AIService.executeWithLogging — logged to LlmLog, billed when the
+   * global provider is costed, and inside the daily autonomous token budget.
+   *
+   * Because metering is universal on this path, a project-owned provider
+   * still wins, but when the project owns none the shared global provider
+   * is returned ON CLOUD TOO — its usage is billed as metered AI tokens.
+   * Cloud zero-config completes: fix tasks work with no per-project
+   * provider. (The old raw-key path — get-llm-config handing the provider
+   * apiKey to the worker for unmetered direct calls — is removed; this is
+   * the only agent provider resolution left.)
+   */
+  @CaptureSpan()
+  public async getLlmProviderForMeteredAgentPath(
+    projectId: ObjectID,
+  ): Promise<Model | null> {
+    const projectOwnedProvider: Model | null =
+      await this.getProjectOwnedLlmProvider(projectId);
+
+    if (projectOwnedProvider) {
+      return projectOwnedProvider;
+    }
+
+    return this.findOneBy({
+      query: {
+        projectId: QueryHelper.isNull(),
+        isGlobalLlm: true,
+      },
+      select: {
+        _id: true,
+        name: true,
+        llmType: true,
+        apiKey: true,
+        baseUrl: true,
+        modelName: true,
+        additionalParams: true,
+        isGlobalLlm: true,
+        costPerMillionTokensInUSDCents: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+
   /*
    * Resolve the provider to use for a chat turn. When the user has explicitly
    * chosen a provider (llmProviderId), use it — but only if it is actually