@oneuptime/common 11.7.3 → 12.0.0

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -474,6 +474,26 @@ import { AddHotQueryIndexes1785140242697 } from "./1785140242697-AddHotQueryInde
 import { AddHotQueryIndexesSecondPass1785148065137 } from "./1785148065137-AddHotQueryIndexesSecondPass";
 import { RepairCrossProjectMonitorStatusReferences1785240000000 } from "./1785240000000-RepairCrossProjectMonitorStatusReferences";
 import { AddColumnsToTableView1785241000000 } from "./1785241000000-AddColumnsToTableView";
+import { RepairCrossProjectIncidentReferences1785320000000 } from "./1785320000000-RepairCrossProjectIncidentReferences";
+import { AddParentGroupToStatusPageGroup1785329453269 } from "./1785329453269-AddParentGroupToStatusPageGroup";
+import { AddRecommendationDismissalTable1785397674289 } from "./1785397674289-AddRecommendationDismissalTable";
+import { AddDroppedCountersToDropFilters1785405581596 } from "./1785405581596-AddDroppedCountersToDropFilters";
+import { AddSessionReplay1785417351021 } from "./1785417351021-AddSessionReplay";
+import { EnableSessionReplayByDefault1785491583874 } from "./1785491583874-EnableSessionReplayByDefault";
+import { AddMonitorSummaryToIncidentAndAlert1785496223183 } from "./1785496223183-AddMonitorSummaryToIncidentAndAlert";
+import { AddSessionReplayReliabilityColumns1785516996695 } from "./1785516996695-AddSessionReplayReliabilityColumns";
+import { AddSessionReplayCorrelationColumns1785533806494 } from "./1785533806494-AddSessionReplayCorrelationColumns";
+import { AddRunnerCapabilities1785776960660 } from "./1785776960660-AddRunnerCapabilities";
+import { AddRedisAndPostgresHealthNotificationColumns1785749674203 } from "./1785749674203-AddRedisAndPostgresHealthNotificationColumns";
+import { AddAutoRemediation1785763818197 } from "./1785763818197-AddAutoRemediation";
+import { RenameRunbookAgentToRunner1785810000000 } from "./1785810000000-RenameRunbookAgentToRunner";
+import { AddRunbookCredential1785800000000 } from "./1785800000000-AddRunbookCredential";
+import { SessionReplayDefaultToSensitiveInputMasking1785860000000 } from "./1785860000000-SessionReplayDefaultToSensitiveInputMasking";
+import { AddAiCommandRemediation1785870000000 } from "./1785870000000-AddAiCommandRemediation";
+import { AlignRunnerSchemaWithEntities1785880000000 } from "./1785880000000-AlignRunnerSchemaWithEntities";
+import { AddFixVerificationAndAutomaticCodeFixes1785890000000 } from "./1785890000000-AddFixVerificationAndAutomaticCodeFixes";
+import { AddIncidentInvestigationGating1785790000000 } from "./1785790000000-AddIncidentInvestigationGating";
+import { AddRemediationVerification1785768089408 } from "./1785768089408-AddRemediationVerification";
 
 export default [
   InitialMigration,
@@ -952,4 +972,24 @@ export default [
   AddHotQueryIndexesSecondPass1785148065137,
   RepairCrossProjectMonitorStatusReferences1785240000000,
   AddColumnsToTableView1785241000000,
+  RepairCrossProjectIncidentReferences1785320000000,
+  AddParentGroupToStatusPageGroup1785329453269,
+  AddRecommendationDismissalTable1785397674289,
+  AddDroppedCountersToDropFilters1785405581596,
+  AddSessionReplay1785417351021,
+  EnableSessionReplayByDefault1785491583874,
+  AddMonitorSummaryToIncidentAndAlert1785496223183,
+  AddSessionReplayReliabilityColumns1785516996695,
+  AddSessionReplayCorrelationColumns1785533806494,
+  AddRedisAndPostgresHealthNotificationColumns1785749674203,
+  AddAutoRemediation1785763818197,
+  AddRunnerCapabilities1785776960660,
+  AddRemediationVerification1785768089408,
+  AddIncidentInvestigationGating1785790000000,
+  AddRunbookCredential1785800000000,
+  RenameRunbookAgentToRunner1785810000000,
+  SessionReplayDefaultToSensitiveInputMasking1785860000000,
+  AddAiCommandRemediation1785870000000,
+  AlignRunnerSchemaWithEntities1785880000000,
+  AddFixVerificationAndAutomaticCodeFixes1785890000000,
 ];

package/Server/Services/AIRunService.ts

@@ -104,6 +104,15 @@ export class Service extends DatabaseService<Model> {
   @CaptureSpan()
   public async claimNextQueuedCodeFixRun(data: {
     aiAgentId: ObjectID;
+    /*
+     * The claiming agent's project, when it is project-scoped. A Runner a
+     * customer installs carries a project-scoped credential and must only
+     * ever see that project's work — without this filter it would claim the
+     * oldest queued run across every tenant. Left undefined only for the
+     * in-cluster Runner (cluster-key registered, no projectId), which
+     * serves all projects by design.
+     */
+    projectId?: ObjectID | undefined;
   }): Promise<Model | null> {
     const maxClaimAttempts: number = 5;
 
@@ -112,6 +121,7 @@ export class Service extends DatabaseService<Model> {
         query: {
           runType: AIRunType.CodeFix,
           status: AIRunStatus.Queued,
+          ...(data.projectId ? { projectId: data.projectId } : {}),
         },
         sort: {
           createdAt: SortOrder.Ascending,

package/Server/Services/AIService.ts

@@ -113,6 +113,24 @@ export const RUNBOOK_AI_STEP_FEATURE: string = "Runbook AI Step";
  */
 export const AI_INSIGHT_TRIAGE_FEATURE: string = "AI Insight Triage";
 
+/*
+ * The LlmLog feature name for auto-remediation planning: the read-only
+ * RemediationPlan AIRun that picks the most applicable runbook for a matched
+ * AI auto-remediation rule. Rules fire on incident/alert creation, so these
+ * calls are storm-shaped — one flapping monitor can queue many plans.
+ */
+export const AI_REMEDIATION_PLANNING_FEATURE: string =
+  "AI Remediation Planning";
+
+/*
+ * RemediationExecution AIRun for a rule with aiComposesCommands: the run
+ * that investigates and composes (and, under a FullAuto allowlist, executes)
+ * remediation commands. Fired by incident/alert creation with no human in
+ * the loop — storm-shaped like planning, and budget-gated like it.
+ */
+export const AI_REMEDIATION_EXECUTION_FEATURE: string =
+  "AI Remediation Execution";
+
 /*
  * The pre-rename values of the six labels that carried the "Sentinel" codename.
  * These are NOT written by anything any more — they exist only so the daily
@@ -188,6 +206,18 @@ export const AUTONOMOUS_AI_FEATURES: Array<string> = [
    * budget must cover these runs too.
    */
   AI_INSIGHT_TRIAGE_FEATURE,
+  /*
+   * Auto-remediation planning (RemediationPlanRunner). One read-only,
+   * constrained call per AI-rule match, fired by incident/alert creation
+   * with no human in the loop — storm-shaped by construction.
+   */
+  AI_REMEDIATION_PLANNING_FEATURE,
+  /*
+   * Auto-remediation command composition/execution
+   * (RemediationExecutionRunner). Same trigger shape as planning, with a
+   * larger per-run tool budget — the daily pool must cover it.
+   */
+  AI_REMEDIATION_EXECUTION_FEATURE,
   /*
    * Pre-rename labels. Keeps the budget honest for rows already persisted
    * under the old names — read the LEGACY_AUTONOMOUS_AI_FEATURES comment

package/Server/Services/AlertEpisodeService.ts

@@ -32,6 +32,9 @@ import {
 import URL from "../../Types/API/URL";
 import DatabaseConfig from "../DatabaseConfig";
 import AlertSeverityService from "./AlertSeverityService";
+import ProjectScopedReferenceValidator, {
+  resolveReferenceId,
+} from "../Utils/Database/ProjectScopedReferenceValidator";
 import AlertEpisodeMemberService from "./AlertEpisodeMemberService";
 import AlertEpisodeOwnerUserService from "./AlertEpisodeOwnerUserService";
 import AlertEpisodeOwnerTeamService from "./AlertEpisodeOwnerTeamService";
@@ -94,6 +97,36 @@ export class Service extends DatabaseService<Model> {
       updateBy.query,
       updateBy.props,
     );
+
+    /*
+     * An episode carries the same project-scoped state and severity an alert
+     * does, on FKs that are equally ON DELETE NO ACTION, so an id from another
+     * project here leaves that project undeletable in exactly the same way.
+     * onBeforeCreate overwrites currentAlertStateId with this project's
+     * created state, which leaves the severity as the only create-reachable
+     * column — but an update can write either.
+     */
+    await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+      projectId: updateBy.props.tenantId,
+      subject: "alert episode",
+      references: [
+        {
+          modelName: "Alert State",
+          id:
+            resolveReferenceId(updateBy.data.currentAlertStateId) ||
+            resolveReferenceId(updateBy.data.currentAlertState),
+          service: AlertStateService,
+        },
+        {
+          modelName: "Alert Severity",
+          id:
+            resolveReferenceId(updateBy.data.alertSeverityId) ||
+            resolveReferenceId(updateBy.data.alertSeverity),
+          service: AlertSeverityService,
+        },
+      ],
+    });
+
     return { updateBy, carryForward: null };
   }
 
@@ -141,6 +174,20 @@ export class Service extends DatabaseService<Model> {
 
     createBy.data.currentAlertStateId = alertState.id;
 
+    await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+      projectId: projectId,
+      subject: "alert episode",
+      references: [
+        {
+          modelName: "Alert Severity",
+          id:
+            resolveReferenceId(createBy.data.alertSeverityId) ||
+            resolveReferenceId(createBy.data.alertSeverity),
+          service: AlertSeverityService,
+        },
+      ],
+    });
+
     // Auto-generate episode number
     const episodeCounterResult: {
       counter: number;

package/Server/Services/AlertService.ts

@@ -29,6 +29,10 @@ import Model from "../../Models/DatabaseModels/Alert";
 import AlertOwnerTeam from "../../Models/DatabaseModels/AlertOwnerTeam";
 import AlertOwnerUser from "../../Models/DatabaseModels/AlertOwnerUser";
 import AlertState from "../../Models/DatabaseModels/AlertState";
+import MonitorStatusService from "./MonitorStatusService";
+import ProjectScopedReferenceValidator, {
+  resolveReferenceId,
+} from "../Utils/Database/ProjectScopedReferenceValidator";
 import AlertStateTimeline from "../../Models/DatabaseModels/AlertStateTimeline";
 import User from "../../Models/DatabaseModels/User";
 import { IsBillingEnabled } from "../EnvironmentConfig";
@@ -67,6 +71,7 @@ import AlertLabelRuleEngineService from "./AlertLabelRuleEngineService";
 import AlertOnCallRuleEngineService from "./AlertOnCallRuleEngineService";
 import AlertOwnerRuleEngineService from "./AlertOwnerRuleEngineService";
 import RunbookRuleEngineService from "./RunbookRuleEngineService";
+import AutoRemediationRuleEngineService from "./AutoRemediationRuleEngineService";
 import AIAlertInvestigationRunner from "../Utils/AI/SRE/AlertInvestigationRunner";
 import AlertPrivacyRuleEngineService from "./AlertPrivacyRuleEngineService";
 import ProjectService from "./ProjectService";
@@ -258,9 +263,98 @@ export class Service extends DatabaseService<Model> {
       updateBy.query,
       updateBy.props,
     );
+
+    await this.validateProjectScopedReferences(updateBy);
+
     return { updateBy, carryForward: null };
   }
 
+  /*
+   * An update can repoint an alert at another project's state, severity or
+   * monitor status just as easily as a create can, and the result is the same:
+   * the referenced project can no longer be deleted. Only the columns actually
+   * being written are checked, so ordinary updates cost no extra queries.
+   */
+  private async validateProjectScopedReferences(
+    updateBy: UpdateBy<Model>,
+  ): Promise<void> {
+    const alertStateId: ObjectID | string | undefined =
+      resolveReferenceId(updateBy.data.currentAlertStateId) ||
+      resolveReferenceId(updateBy.data.currentAlertState);
+
+    const alertSeverityId: ObjectID | string | undefined =
+      resolveReferenceId(updateBy.data.alertSeverityId) ||
+      resolveReferenceId(updateBy.data.alertSeverity);
+
+    const monitorStatusId: ObjectID | string | undefined =
+      resolveReferenceId(
+        updateBy.data.monitorStatusWhenThisAlertWasCreatedId,
+      ) ||
+      resolveReferenceId(updateBy.data.monitorStatusWhenThisAlertWasCreated);
+
+    if (!alertStateId && !alertSeverityId && !monitorStatusId) {
+      return;
+    }
+
+    /*
+     * Root/API updates do not always carry a tenantId, so fall back to the
+     * project of each alert the query actually matches.
+     */
+    const projectIds: Array<ObjectID> = updateBy.props.tenantId
+      ? [updateBy.props.tenantId]
+      : await this.getProjectIdsForUpdateQuery(updateBy);
+
+    for (const projectId of projectIds) {
+      await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: projectId,
+        subject: "alert",
+        references: [
+          {
+            modelName: "Alert State",
+            id: alertStateId,
+            service: AlertStateService,
+          },
+          {
+            modelName: "Alert Severity",
+            id: alertSeverityId,
+            service: AlertSeverityService,
+          },
+          {
+            modelName: "Monitor Status",
+            id: monitorStatusId,
+            service: MonitorStatusService,
+          },
+        ],
+      });
+    }
+  }
+
+  private async getProjectIdsForUpdateQuery(
+    updateBy: UpdateBy<Model>,
+  ): Promise<Array<ObjectID>> {
+    const alerts: Array<Model> = await this.findBy({
+      query: updateBy.query,
+      select: {
+        projectId: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    const projectIds: Dictionary<ObjectID> = {};
+
+    for (const alert of alerts) {
+      if (alert.projectId) {
+        projectIds[alert.projectId.toString()] = alert.projectId;
+      }
+    }
+
+    return Object.values(projectIds);
+  }
+
   @CaptureSpan()
   protected override async onBeforeCreate(
     createBy: CreateBy<Model>,
@@ -293,6 +387,38 @@ export class Service extends DatabaseService<Model> {
 
     createBy.data.currentAlertStateId = alertState.id;
 
+    /*
+     * The severity and the monitor status stamped on the alert come from the
+     * monitor criteria or the API caller, neither of which checked that the
+     * record belongs to this project. Persisting another project's id leaves
+     * that project undeletable. Runs before the counter increment so a
+     * rejected create does not burn an alert number.
+     */
+    await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+      projectId: projectId,
+      subject: "alert",
+      references: [
+        {
+          modelName: "Alert Severity",
+          id:
+            resolveReferenceId(createBy.data.alertSeverityId) ||
+            resolveReferenceId(createBy.data.alertSeverity),
+          service: AlertSeverityService,
+        },
+        {
+          modelName: "Monitor Status",
+          id:
+            resolveReferenceId(
+              createBy.data.monitorStatusWhenThisAlertWasCreatedId,
+            ) ||
+            resolveReferenceId(
+              createBy.data.monitorStatusWhenThisAlertWasCreated,
+            ),
+          service: MonitorStatusService,
+        },
+      ],
+    });
+
     const alertCounterResult: {
       counter: number;
       prefix: string | undefined;
@@ -349,6 +475,14 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("currentAlertStateId is required");
     }
 
+    /*
+     * Whether an AI investigation run was enqueued for this alert — set by
+     * the investigation step below and read by the auto-remediation step
+     * after it: an enqueued investigation DEFERS remediation until the run
+     * settles (RCA-first ordering, see RemediationHandoff).
+     */
+    let aiInvestigationEnqueued: boolean = false;
+
     // Execute operations sequentially with error handling
     Promise.resolve()
       .then(async () => {
@@ -568,10 +702,11 @@ export class Service extends DatabaseService<Model> {
          */
         try {
           if (createdItem.projectId && createdItem.id) {
-            await AIAlertInvestigationRunner.investigateNewAlert({
-              alertId: createdItem.id,
-              projectId: createdItem.projectId,
-            });
+            aiInvestigationEnqueued =
+              await AIAlertInvestigationRunner.investigateNewAlert({
+                alertId: createdItem.id,
+                projectId: createdItem.projectId,
+              });
           }
         } catch (error) {
           logger.error(
@@ -583,6 +718,33 @@ export class Service extends DatabaseService<Model> {
           );
         }
       })
+      .then(async () => {
+        /*
+         * Auto-remediation runs LAST — and only when NO AI investigation
+         * was enqueued above. RCA-first ordering: when an investigation is
+         * in flight, remediation is deferred until that run settles
+         * (RemediationHandoff releases it on any terminal outcome), so the
+         * remediation planner always has the posted root cause analysis as
+         * input instead of racing it. Without an investigation (opt-out,
+         * gates, budget) remediation fires here immediately — it must
+         * never silently depend on the AI lane being enabled.
+         */
+        try {
+          if (!aiInvestigationEnqueued) {
+            await AutoRemediationRuleEngineService.applyRulesToAlert(
+              createdItem,
+            );
+          }
+        } catch (error) {
+          logger.error(
+            `Apply auto-remediation rules failed in AlertService.onCreateSuccess: ${error}`,
+            {
+              projectId: createdItem.projectId?.toString(),
+              alertId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+        }
+      })
       .catch((error: Error) => {
         logger.error(
           `Critical error in AlertService sequential operations: ${error}`,

package/Server/Services/AlertSeverityService.ts

@@ -1,3 +1,4 @@
+import logger from "../Utils/Logger";
 import CreateBy from "../Types/Database/CreateBy";
 import DeleteBy from "../Types/Database/DeleteBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
@@ -141,17 +142,30 @@ export class Service extends DatabaseService<Model> {
         newOrder = alertSeverity.order! - 1;
       }
 
-      await this.updateOneBy({
-        query: {
-          _id: alertSeverity._id!,
-        },
-        data: {
-          order: newOrder,
-        },
-        props: {
-          isRoot: true,
-        },
-      });
+      /*
+       * Concurrent deletes (e.g. Terraform destroying several items in
+       * parallel) can soft-delete a row between the findBy above and
+       * this update; save() would then INSERT with a null projectId and
+       * fail the whole delete with a 500. A row that vanished
+       * mid-rearrange needs no repositioning - skip it.
+       */
+      try {
+        await this.updateOneBy({
+          query: {
+            _id: alertSeverity._id!,
+          },
+          data: {
+            order: newOrder,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      } catch (err) {
+        logger.warn(
+          `rearrange: skipping row (likely deleted concurrently): ${err}`,
+        );
+      }
     }
   }
 }

package/Server/Services/AlertStateService.ts

@@ -1,3 +1,4 @@
+import logger from "../Utils/Logger";
 import CreateBy from "../Types/Database/CreateBy";
 import DeleteBy from "../Types/Database/DeleteBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
@@ -143,17 +144,30 @@ export class Service extends DatabaseService<AlertState> {
         newOrder = alertState.order! - 1;
       }
 
-      await this.updateOneBy({
-        query: {
-          _id: alertState._id!,
-        },
-        data: {
-          order: newOrder,
-        },
-        props: {
-          isRoot: true,
-        },
-      });
+      /*
+       * Concurrent deletes (e.g. Terraform destroying several items in
+       * parallel) can soft-delete a row between the findBy above and
+       * this update; save() would then INSERT with a null projectId and
+       * fail the whole delete with a 500. A row that vanished
+       * mid-rearrange needs no repositioning - skip it.
+       */
+      try {
+        await this.updateOneBy({
+          query: {
+            _id: alertState._id!,
+          },
+          data: {
+            order: newOrder,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      } catch (err) {
+        logger.warn(
+          `rearrange: skipping row (likely deleted concurrently): ${err}`,
+        );
+      }
     }
   }