@oneuptime/common 11.7.4 → 12.0.0

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -478,6 +478,22 @@ import { RepairCrossProjectIncidentReferences1785320000000 } from "./17853200000
 import { AddParentGroupToStatusPageGroup1785329453269 } from "./1785329453269-AddParentGroupToStatusPageGroup";
 import { AddRecommendationDismissalTable1785397674289 } from "./1785397674289-AddRecommendationDismissalTable";
 import { AddDroppedCountersToDropFilters1785405581596 } from "./1785405581596-AddDroppedCountersToDropFilters";
+import { AddSessionReplay1785417351021 } from "./1785417351021-AddSessionReplay";
+import { EnableSessionReplayByDefault1785491583874 } from "./1785491583874-EnableSessionReplayByDefault";
+import { AddMonitorSummaryToIncidentAndAlert1785496223183 } from "./1785496223183-AddMonitorSummaryToIncidentAndAlert";
+import { AddSessionReplayReliabilityColumns1785516996695 } from "./1785516996695-AddSessionReplayReliabilityColumns";
+import { AddSessionReplayCorrelationColumns1785533806494 } from "./1785533806494-AddSessionReplayCorrelationColumns";
+import { AddRunnerCapabilities1785776960660 } from "./1785776960660-AddRunnerCapabilities";
+import { AddRedisAndPostgresHealthNotificationColumns1785749674203 } from "./1785749674203-AddRedisAndPostgresHealthNotificationColumns";
+import { AddAutoRemediation1785763818197 } from "./1785763818197-AddAutoRemediation";
+import { RenameRunbookAgentToRunner1785810000000 } from "./1785810000000-RenameRunbookAgentToRunner";
+import { AddRunbookCredential1785800000000 } from "./1785800000000-AddRunbookCredential";
+import { SessionReplayDefaultToSensitiveInputMasking1785860000000 } from "./1785860000000-SessionReplayDefaultToSensitiveInputMasking";
+import { AddAiCommandRemediation1785870000000 } from "./1785870000000-AddAiCommandRemediation";
+import { AlignRunnerSchemaWithEntities1785880000000 } from "./1785880000000-AlignRunnerSchemaWithEntities";
+import { AddFixVerificationAndAutomaticCodeFixes1785890000000 } from "./1785890000000-AddFixVerificationAndAutomaticCodeFixes";
+import { AddIncidentInvestigationGating1785790000000 } from "./1785790000000-AddIncidentInvestigationGating";
+import { AddRemediationVerification1785768089408 } from "./1785768089408-AddRemediationVerification";
 
 export default [
   InitialMigration,
@@ -960,4 +976,20 @@ export default [
   AddParentGroupToStatusPageGroup1785329453269,
   AddRecommendationDismissalTable1785397674289,
   AddDroppedCountersToDropFilters1785405581596,
+  AddSessionReplay1785417351021,
+  EnableSessionReplayByDefault1785491583874,
+  AddMonitorSummaryToIncidentAndAlert1785496223183,
+  AddSessionReplayReliabilityColumns1785516996695,
+  AddSessionReplayCorrelationColumns1785533806494,
+  AddRedisAndPostgresHealthNotificationColumns1785749674203,
+  AddAutoRemediation1785763818197,
+  AddRunnerCapabilities1785776960660,
+  AddRemediationVerification1785768089408,
+  AddIncidentInvestigationGating1785790000000,
+  AddRunbookCredential1785800000000,
+  RenameRunbookAgentToRunner1785810000000,
+  SessionReplayDefaultToSensitiveInputMasking1785860000000,
+  AddAiCommandRemediation1785870000000,
+  AlignRunnerSchemaWithEntities1785880000000,
+  AddFixVerificationAndAutomaticCodeFixes1785890000000,
 ];

package/Server/Services/AIRunService.ts

@@ -104,6 +104,15 @@ export class Service extends DatabaseService<Model> {
   @CaptureSpan()
   public async claimNextQueuedCodeFixRun(data: {
     aiAgentId: ObjectID;
+    /*
+     * The claiming agent's project, when it is project-scoped. A Runner a
+     * customer installs carries a project-scoped credential and must only
+     * ever see that project's work — without this filter it would claim the
+     * oldest queued run across every tenant. Left undefined only for the
+     * in-cluster Runner (cluster-key registered, no projectId), which
+     * serves all projects by design.
+     */
+    projectId?: ObjectID | undefined;
   }): Promise<Model | null> {
     const maxClaimAttempts: number = 5;
 
@@ -112,6 +121,7 @@ export class Service extends DatabaseService<Model> {
         query: {
           runType: AIRunType.CodeFix,
           status: AIRunStatus.Queued,
+          ...(data.projectId ? { projectId: data.projectId } : {}),
         },
         sort: {
           createdAt: SortOrder.Ascending,

package/Server/Services/AIService.ts

@@ -113,6 +113,24 @@ export const RUNBOOK_AI_STEP_FEATURE: string = "Runbook AI Step";
  */
 export const AI_INSIGHT_TRIAGE_FEATURE: string = "AI Insight Triage";
 
+/*
+ * The LlmLog feature name for auto-remediation planning: the read-only
+ * RemediationPlan AIRun that picks the most applicable runbook for a matched
+ * AI auto-remediation rule. Rules fire on incident/alert creation, so these
+ * calls are storm-shaped — one flapping monitor can queue many plans.
+ */
+export const AI_REMEDIATION_PLANNING_FEATURE: string =
+  "AI Remediation Planning";
+
+/*
+ * RemediationExecution AIRun for a rule with aiComposesCommands: the run
+ * that investigates and composes (and, under a FullAuto allowlist, executes)
+ * remediation commands. Fired by incident/alert creation with no human in
+ * the loop — storm-shaped like planning, and budget-gated like it.
+ */
+export const AI_REMEDIATION_EXECUTION_FEATURE: string =
+  "AI Remediation Execution";
+
 /*
  * The pre-rename values of the six labels that carried the "Sentinel" codename.
  * These are NOT written by anything any more — they exist only so the daily
@@ -188,6 +206,18 @@ export const AUTONOMOUS_AI_FEATURES: Array<string> = [
    * budget must cover these runs too.
    */
   AI_INSIGHT_TRIAGE_FEATURE,
+  /*
+   * Auto-remediation planning (RemediationPlanRunner). One read-only,
+   * constrained call per AI-rule match, fired by incident/alert creation
+   * with no human in the loop — storm-shaped by construction.
+   */
+  AI_REMEDIATION_PLANNING_FEATURE,
+  /*
+   * Auto-remediation command composition/execution
+   * (RemediationExecutionRunner). Same trigger shape as planning, with a
+   * larger per-run tool budget — the daily pool must cover it.
+   */
+  AI_REMEDIATION_EXECUTION_FEATURE,
   /*
    * Pre-rename labels. Keeps the budget honest for rows already persisted
    * under the old names — read the LEGACY_AUTONOMOUS_AI_FEATURES comment

package/Server/Services/AlertService.ts

@@ -71,6 +71,7 @@ import AlertLabelRuleEngineService from "./AlertLabelRuleEngineService";
 import AlertOnCallRuleEngineService from "./AlertOnCallRuleEngineService";
 import AlertOwnerRuleEngineService from "./AlertOwnerRuleEngineService";
 import RunbookRuleEngineService from "./RunbookRuleEngineService";
+import AutoRemediationRuleEngineService from "./AutoRemediationRuleEngineService";
 import AIAlertInvestigationRunner from "../Utils/AI/SRE/AlertInvestigationRunner";
 import AlertPrivacyRuleEngineService from "./AlertPrivacyRuleEngineService";
 import ProjectService from "./ProjectService";
@@ -474,6 +475,14 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("currentAlertStateId is required");
     }
 
+    /*
+     * Whether an AI investigation run was enqueued for this alert — set by
+     * the investigation step below and read by the auto-remediation step
+     * after it: an enqueued investigation DEFERS remediation until the run
+     * settles (RCA-first ordering, see RemediationHandoff).
+     */
+    let aiInvestigationEnqueued: boolean = false;
+
     // Execute operations sequentially with error handling
     Promise.resolve()
       .then(async () => {
@@ -693,10 +702,11 @@ export class Service extends DatabaseService<Model> {
          */
         try {
           if (createdItem.projectId && createdItem.id) {
-            await AIAlertInvestigationRunner.investigateNewAlert({
-              alertId: createdItem.id,
-              projectId: createdItem.projectId,
-            });
+            aiInvestigationEnqueued =
+              await AIAlertInvestigationRunner.investigateNewAlert({
+                alertId: createdItem.id,
+                projectId: createdItem.projectId,
+              });
           }
         } catch (error) {
           logger.error(
@@ -708,6 +718,33 @@ export class Service extends DatabaseService<Model> {
           );
         }
       })
+      .then(async () => {
+        /*
+         * Auto-remediation runs LAST — and only when NO AI investigation
+         * was enqueued above. RCA-first ordering: when an investigation is
+         * in flight, remediation is deferred until that run settles
+         * (RemediationHandoff releases it on any terminal outcome), so the
+         * remediation planner always has the posted root cause analysis as
+         * input instead of racing it. Without an investigation (opt-out,
+         * gates, budget) remediation fires here immediately — it must
+         * never silently depend on the AI lane being enabled.
+         */
+        try {
+          if (!aiInvestigationEnqueued) {
+            await AutoRemediationRuleEngineService.applyRulesToAlert(
+              createdItem,
+            );
+          }
+        } catch (error) {
+          logger.error(
+            `Apply auto-remediation rules failed in AlertService.onCreateSuccess: ${error}`,
+            {
+              projectId: createdItem.projectId?.toString(),
+              alertId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+        }
+      })
       .catch((error: Error) => {
         logger.error(
           `Critical error in AlertService sequential operations: ${error}`,

package/Server/Services/AlertSeverityService.ts

@@ -1,3 +1,4 @@
+import logger from "../Utils/Logger";
 import CreateBy from "../Types/Database/CreateBy";
 import DeleteBy from "../Types/Database/DeleteBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
@@ -141,17 +142,30 @@ export class Service extends DatabaseService<Model> {
         newOrder = alertSeverity.order! - 1;
       }
 
-      await this.updateOneBy({
-        query: {
-          _id: alertSeverity._id!,
-        },
-        data: {
-          order: newOrder,
-        },
-        props: {
-          isRoot: true,
-        },
-      });
+      /*
+       * Concurrent deletes (e.g. Terraform destroying several items in
+       * parallel) can soft-delete a row between the findBy above and
+       * this update; save() would then INSERT with a null projectId and
+       * fail the whole delete with a 500. A row that vanished
+       * mid-rearrange needs no repositioning - skip it.
+       */
+      try {
+        await this.updateOneBy({
+          query: {
+            _id: alertSeverity._id!,
+          },
+          data: {
+            order: newOrder,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      } catch (err) {
+        logger.warn(
+          `rearrange: skipping row (likely deleted concurrently): ${err}`,
+        );
+      }
     }
   }
 }

package/Server/Services/AlertStateService.ts

@@ -1,3 +1,4 @@
+import logger from "../Utils/Logger";
 import CreateBy from "../Types/Database/CreateBy";
 import DeleteBy from "../Types/Database/DeleteBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
@@ -143,17 +144,30 @@ export class Service extends DatabaseService<AlertState> {
         newOrder = alertState.order! - 1;
       }
 
-      await this.updateOneBy({
-        query: {
-          _id: alertState._id!,
-        },
-        data: {
-          order: newOrder,
-        },
-        props: {
-          isRoot: true,
-        },
-      });
+      /*
+       * Concurrent deletes (e.g. Terraform destroying several items in
+       * parallel) can soft-delete a row between the findBy above and
+       * this update; save() would then INSERT with a null projectId and
+       * fail the whole delete with a 500. A row that vanished
+       * mid-rearrange needs no repositioning - skip it.
+       */
+      try {
+        await this.updateOneBy({
+          query: {
+            _id: alertState._id!,
+          },
+          data: {
+            order: newOrder,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      } catch (err) {
+        logger.warn(
+          `rearrange: skipping row (likely deleted concurrently): ${err}`,
+        );
+      }
     }
   }