@oneuptime/common 11.5.12 → 11.6.0

package/Server/Types/Workflow/Components/Schedule.ts

@@ -15,6 +15,7 @@ import ComponentID from "../../../../Types/Workflow/ComponentID";
 import ScheduleComponents from "../../../../Types/Workflow/Components/Schedule";
 import Workflow from "../../../../Models/DatabaseModels/Workflow";
 import CaptureSpan from "../../../Utils/Telemetry/CaptureSpan";
+import logger from "../../../Utils/Logger";
 
 export default class WebhookTrigger extends TriggerCode {
   public constructor() {
@@ -52,24 +53,37 @@ export default class WebhookTrigger extends TriggerCode {
 
     // query all workflows.
     for (const workflow of workflows) {
-      const executeWorkflow: ExecuteWorkflowType = {
-        workflowId: new ObjectID(workflow._id!),
-        returnValues: {},
-      };
-
-      if (
-        workflow.triggerArguments &&
-        workflow.triggerArguments["schedule"] &&
-        workflow.isEnabled
-      ) {
-        await props.scheduleWorkflow(
-          executeWorkflow,
-          workflow.triggerArguments["schedule"] as string,
+      /*
+       * Isolate each workflow: a single workflow whose schedule can't be
+       * registered (e.g. an invalid cron, or a transient queue error) must not
+       * abort the startup scan and leave every other scheduled workflow
+       * unregistered.
+       */
+      try {
+        const executeWorkflow: ExecuteWorkflowType = {
+          workflowId: new ObjectID(workflow._id!),
+          returnValues: {},
+        };
+
+        if (
+          workflow.triggerArguments &&
+          workflow.triggerArguments["schedule"] &&
+          workflow.isEnabled
+        ) {
+          await props.scheduleWorkflow(
+            executeWorkflow,
+            workflow.triggerArguments["schedule"] as string,
+          );
+        }
+
+        if (!workflow.isEnabled) {
+          await props.removeWorkflow(workflow.id!);
+        }
+      } catch (err) {
+        logger.error(
+          `Failed to register schedule for workflow ${workflow._id}`,
         );
-      }
-
-      if (!workflow.isEnabled) {
-        await props.removeWorkflow(workflow.id!);
+        logger.error(err);
       }
     }
   }
@@ -128,23 +142,34 @@ export default class WebhookTrigger extends TriggerCode {
       returnValues: {},
     };
 
-    if (
-      workflow.triggerArguments &&
-      workflow.triggerArguments["schedule"] &&
-      workflow.isEnabled
-    ) {
-      await this.scheduleWorkflow(
-        executeWorkflow,
-        workflow.triggerArguments["schedule"] as string,
-      );
-    }
+    /*
+     * Guard the (re)registration so a bad schedule can't fail the workflow
+     * save that triggered this update. Invalid crons are already surfaced as a
+     * workflow log by the queue layer; this catch is a backstop for unexpected
+     * errors (e.g. a transient queue error).
+     */
+    try {
+      if (
+        workflow.triggerArguments &&
+        workflow.triggerArguments["schedule"] &&
+        workflow.isEnabled
+      ) {
+        await this.scheduleWorkflow(
+          executeWorkflow,
+          workflow.triggerArguments["schedule"] as string,
+        );
+      }
 
-    if (!this.removeWorkflow) {
-      return;
-    }
+      if (!this.removeWorkflow) {
+        return;
+      }
 
-    if (!workflow.isEnabled) {
-      await this.removeWorkflow(workflow.id!);
+      if (!workflow.isEnabled) {
+        await this.removeWorkflow(workflow.id!);
+      }
+    } catch (err) {
+      logger.error(`Failed to update schedule for workflow ${workflow._id}`);
+      logger.error(err);
     }
   }
 }

package/Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.ts

@@ -13,6 +13,7 @@ import TelemetryExceptionService from "../../../../../Services/TelemetryExceptio
 import ExceptionInstanceService from "../../../../../Services/ExceptionInstanceService";
 import ServiceService from "../../../../../Services/ServiceService";
 import QueryHelper from "../../../../../Types/Database/QueryHelper";
+import { sanitizeExceptionMessage } from "../../../../Telemetry/ExceptionSanitizer";
 import {
   InsightCandidate,
   InsightDetector,
@@ -239,9 +240,19 @@ export default class ExceptionSpikeDetector implements InsightDetector {
         exception.primaryEntityId,
       );
 
+      /*
+       * The message flows into the insight title/detail/evidence, and from
+       * there into the triage LLM prompt — sanitize it once here so no
+       * interpolated user data (IDs, emails, domains) rides along. The raw
+       * message stays on the exception row for the dashboard.
+       */
+      const safeMessage: string = sanitizeExceptionMessage(
+        exception.message || "",
+      );
+
       const label: string = ExceptionSpikeDetector.buildExceptionLabel(
         exception.exceptionType,
-        exception.message,
+        safeMessage,
       );
 
       const firstSeenAtIso: string | undefined = exception.firstSeenAt
@@ -255,8 +266,8 @@ export default class ExceptionSpikeDetector implements InsightDetector {
         "",
         `- Exception: \`${label}\``,
       ];
-      if (exception.message) {
-        detailLines.push(`- Message: ${exception.message}`);
+      if (safeMessage) {
+        detailLines.push(`- Message: ${safeMessage}`);
       }
       if (serviceName) {
         detailLines.push(`- Service: ${serviceName}`);
@@ -281,7 +292,7 @@ export default class ExceptionSpikeDetector implements InsightDetector {
         telemetryExceptionId: exception.id,
         evidence: {
           exception: {
-            exceptionMessage: exception.message,
+            exceptionMessage: safeMessage,
             exceptionType: exception.exceptionType,
             recentOccurrenceCount: recentCount.toNumber(),
             baselineHourlyAverage: decision.baselineHourlyAverage,

package/Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.ts

@@ -9,6 +9,7 @@ import Service from "../../../../../../Models/DatabaseModels/Service";
 import TelemetryExceptionService from "../../../../../Services/TelemetryExceptionService";
 import ServiceService from "../../../../../Services/ServiceService";
 import QueryHelper from "../../../../../Types/Database/QueryHelper";
+import { sanitizeExceptionMessage } from "../../../../Telemetry/ExceptionSanitizer";
 import {
   InsightCandidate,
   InsightDetector,
@@ -166,9 +167,19 @@ export default class NewExceptionDetector implements InsightDetector {
         exception.primaryEntityId,
       );
 
+      /*
+       * The message flows into the insight title/detail/evidence, and from
+       * there into the triage LLM prompt — sanitize it once here so no
+       * interpolated user data (IDs, emails, domains) rides along. The raw
+       * message stays on the exception row for the dashboard.
+       */
+      const safeMessage: string = sanitizeExceptionMessage(
+        exception.message || "",
+      );
+
       const label: string = NewExceptionDetector.buildExceptionLabel(
         exception.exceptionType,
-        exception.message,
+        safeMessage,
       );
 
       const firstSeenAtIso: string | undefined = exception.firstSeenAt
@@ -180,8 +191,8 @@ export default class NewExceptionDetector implements InsightDetector {
         "",
         `- Exception: \`${label}\``,
       ];
-      if (exception.message) {
-        detailLines.push(`- Message: ${exception.message}`);
+      if (safeMessage) {
+        detailLines.push(`- Message: ${safeMessage}`);
       }
       if (serviceName) {
         detailLines.push(`- Service: ${serviceName}`);
@@ -204,7 +215,7 @@ export default class NewExceptionDetector implements InsightDetector {
         telemetryExceptionId: exception.id,
         evidence: {
           exception: {
-            exceptionMessage: exception.message,
+            exceptionMessage: safeMessage,
             exceptionType: exception.exceptionType,
             totalOccurrenceCount: occurrenceCount,
             firstSeenAt: firstSeenAtIso,

package/Server/Utils/AI/SRE/Insights/InsightScanner.ts

@@ -11,6 +11,12 @@ import InsightStore, { UpsertCandidatesResult } from "./InsightStore";
 import InsightFixRouting, { InsightFixRoutingResult } from "./FixRouting";
 import InsightTriage, { InsightTriageResult } from "./Triage";
 import { InsightCandidate, InsightDetector, InsightScanContext } from "./Types";
+import AIInsightType from "../../../../../Types/AI/AIInsightType";
+import AIRunType from "../../../../../Types/AI/AIRunType";
+import AIRunStatus from "../../../../../Types/AI/AIRunStatus";
+import ExceptionAIClassification from "../../../../../Types/AI/ExceptionAIClassification";
+import AIRunService from "../../../../Services/AIRunService";
+import QueryHelper from "../../../../Types/Database/QueryHelper";
 import logger from "../../../Logger";
 import CaptureSpan from "../../../Telemetry/CaptureSpan";
 
@@ -19,8 +25,9 @@ import CaptureSpan from "../../../Telemetry/CaptureSpan";
  *
  * THE RULE: no LLM here. Detectors are deterministic statistical sensors;
  * the LLM only engages per-finding AFTERWARDS through InsightTriage
- * (budgeted, read-only), and the fix decision inside InsightFixRouting is
- * deterministic and reuses the existing budgeted CodeFix creation paths.
+ * (budgeted, read-only). The automatic fix decision happens after triage
+ * completes (InsightTriageRunner → InsightFixRouting) and only for
+ * insights the triage classified as code faults.
  *
  * Everything is opt-in and quiet: only projects with the default-false
  * enableAiInsights flag are scanned, insights never page and never
@@ -73,8 +80,8 @@ export default class InsightScanner {
    * One project's scan: self-heal any insight an earlier tick left unrouted,
    * run every registered detector (isolated — one failing sensor must not
    * blind the others), dedupe/upsert the candidates through InsightStore,
-   * then route each NEWLY created insight to FixOpened or ActionRequired and
-   * enqueue its triage.
+   * then route each NEWLY created insight to ActionRequired and enqueue its
+   * triage (which, on a code-fault verdict, routes the fix).
    */
   @CaptureSpan()
   public static async scanProjectForInsights(project: Project): Promise<void> {
@@ -211,19 +218,200 @@ export default class InsightScanner {
         );
       }
     }
+
+    await this.resweepUntriagedInsights({ projectId });
+    await this.resweepUnroutedCodeFaults({ projectId, project });
+  }
+
+  /*
+   * Stranded shape 2: ActionRequired insights that never got a triage
+   * verdict. With triage gating the automatic fix, a triage that was
+   * quiet-skipped (token budget exhausted), expired in the queue, or
+   * failed would otherwise strand the insight verdict-less FOREVER — and
+   * with it any automatic fix. Re-enqueueing is safe and cheap: the
+   * triage enqueue dedupes to one non-terminal run per insight and
+   * quiet-skips when the budget still has no headroom. Bounded two ways:
+   * only insights from the last 7 days (older rows predate the verdict
+   * gate or were consciously left alone), and only while fewer than 3
+   * triage attempts have ended in failure (a permanently failing triage
+   * must not retry every 15 minutes for a week).
+   */
+  private static async resweepUntriagedInsights(data: {
+    projectId: ObjectID;
+  }): Promise<void> {
+    const { projectId } = data;
+
+    try {
+      const untriaged: Array<AIInsight> = await AIInsightService.findBy({
+        query: {
+          projectId: projectId,
+          status: AIInsightStatus.ActionRequired,
+          triageCompletedAt: QueryHelper.isNull(),
+          createdAt: QueryHelper.greaterThanEqualTo(
+            OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -7),
+          ),
+        },
+        select: {
+          _id: true,
+          projectId: true,
+          insightType: true,
+          serviceName: true,
+          traceId: true,
+          telemetryExceptionId: true,
+          evidence: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: { isRoot: true },
+      });
+
+      for (const insight of untriaged) {
+        try {
+          const failedTriageAttempts: number = (
+            await AIRunService.countBy({
+              query: {
+                runType: AIRunType.Investigation,
+                triggeredByAiInsightId: insight.id!,
+                status: QueryHelper.any([
+                  AIRunStatus.Error,
+                  AIRunStatus.Cancelled,
+                  AIRunStatus.Stale,
+                ]),
+              },
+              props: { isRoot: true },
+            })
+          ).toNumber();
+
+          if (failedTriageAttempts >= 3) {
+            continue;
+          }
+
+          const triageResult: InsightTriageResult =
+            await InsightTriage.enqueueInsightTriage({ insight: insight });
+
+          if (triageResult.triageAiRunId) {
+            await AIInsightService.updateOneById({
+              id: insight.id!,
+              data: {
+                triageAiRunId: triageResult.triageAiRunId,
+              },
+              props: { isRoot: true },
+            });
+          }
+        } catch (error) {
+          logger.error(
+            `AI Insights: triage re-enqueue failed for insight ${insight.id?.toString()} — continuing: ${error}`,
+          );
+        }
+      }
+    } catch (error) {
+      logger.error(
+        `AI Insights: failed to sweep untriaged insights for project ${projectId.toString()} — continuing with the scan: ${error}`,
+      );
+    }
+  }
+
+  /*
+   * Stranded shape 3: a code-fault verdict was recorded but the fix was
+   * never routed — the pod died between the verdict write and the fix
+   * routing, or routing quiet-skipped on a transient gate (daily fix
+   * budget, readiness). Re-routing is idempotent: the creation path's
+   * per-(exception, recipe) dedupe and the cross-group guard refuse
+   * duplicates, and the conditional status flip only moves
+   * ActionRequired → FixOpened. Bounded to verdicts from the last 24
+   * hours so a permanently refused group (declined by a human, capped
+   * repo) does not retry forever. Latency insights are excluded — they
+   * are fix-routed deterministically at scan time.
+   */
+  private static async resweepUnroutedCodeFaults(data: {
+    projectId: ObjectID;
+    project: Project;
+  }): Promise<void> {
+    const { projectId, project } = data;
+
+    try {
+      const unrouted: Array<AIInsight> = await AIInsightService.findBy({
+        query: {
+          projectId: projectId,
+          status: AIInsightStatus.ActionRequired,
+          classification: ExceptionAIClassification.CodeFault,
+          fixAiRunId: QueryHelper.isNull(),
+          triageCompletedAt: QueryHelper.greaterThanEqualTo(
+            OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -1),
+          ),
+        },
+        select: {
+          _id: true,
+          projectId: true,
+          insightType: true,
+          serviceName: true,
+          traceId: true,
+          telemetryExceptionId: true,
+          evidence: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: { isRoot: true },
+      });
+
+      for (const insight of unrouted) {
+        if (insight.insightType === AIInsightType.TraceLatencyRegression) {
+          continue;
+        }
+
+        try {
+          const fixResult: InsightFixRoutingResult =
+            await InsightFixRouting.routeInsightFix({
+              insight: insight,
+              project: project,
+            });
+
+          if (fixResult.fixAiRunId) {
+            await AIInsightService.updateOneBy({
+              query: {
+                _id: insight.id!.toString(),
+                status: AIInsightStatus.ActionRequired,
+              },
+              data: {
+                status: AIInsightStatus.FixOpened,
+                fixAiRunId: fixResult.fixAiRunId,
+              },
+              props: { isRoot: true },
+            });
+          }
+        } catch (error) {
+          logger.error(
+            `AI Insights: fix re-routing failed for insight ${insight.id?.toString()} — continuing: ${error}`,
+          );
+        }
+      }
+    } catch (error) {
+      logger.error(
+        `AI Insights: failed to sweep unrouted code-fault insights for project ${projectId.toString()} — continuing with the scan: ${error}`,
+      );
+    }
   }
 
   /*
    * Route one insight — newly created this tick, or swept up as stranded
-   * from an earlier one — out of the defensive Detected state:
-   *   1. deterministic fix routing (gated on enableAi + enableInsightFixTasks
-   *      inside InsightFixRouting) → FixOpened + fixAiRunId, else
-   *      ActionRequired;
-   *   2. triage enqueue for EVERY new insight — triage enriches fix and
-   *      non-fix insights alike → triageAiRunId when a run was enqueued.
-   * Both collaborators are never-throws by contract, but they are wrapped
-   * anyway: a contract breach must degrade (no fix / no triage), not leave
-   * the insight stuck in Detected or fail the scan.
+   * from an earlier one — out of the defensive Detected state.
+   *
+   * TRIAGE GATES THE FIX. The automatic fix decision no longer happens
+   * here: every new insight goes to ActionRequired and gets its triage
+   * enqueued; when the triage run completes with a code-fault
+   * classification, InsightTriageRunner routes the fix (via
+   * InsightFixRouting) and flips the insight to FixOpened. This exists
+   * because roughly half of real exception insights turn out to be
+   * expected user errors or intentional denials (auth, paywalls, scanner
+   * probes) — opening an unreviewed fix PR for those is worse than
+   * opening none: the agent's only "fix" is to weaken the check that
+   * correctly fired. Fail-closed by design: no triage verdict, no
+   * automatic PR. The manual "Fix with AI" button on the exception page
+   * is unaffected.
+   *
+   * The collaborator is never-throws by contract, but it is wrapped
+   * anyway: a contract breach must degrade (no triage), not leave the
+   * insight stuck in Detected or fail the scan.
    */
   private static async routeNewInsight(data: {
     insight: AIInsight;
@@ -231,19 +419,29 @@ export default class InsightScanner {
   }): Promise<void> {
     const { insight, project } = data;
 
+    /*
+     * TraceLatencyRegression keeps DETERMINISTIC scan-time fix routing:
+     * its evidence is the SpanTreeAnalyzer's own findings (N+1, dominant
+     * span, sequential fan-out) — no LLM verdict adds signal there, and
+     * the exception-centric classification taxonomy would mislabel most
+     * latency regressions as "infrastructure". Exception insights go
+     * through the verdict gate instead.
+     */
     let fixAiRunId: ObjectID | undefined = undefined;
 
-    try {
-      const fixResult: InsightFixRoutingResult =
-        await InsightFixRouting.routeInsightFix({
-          insight: insight,
-          project: project,
-        });
-      fixAiRunId = fixResult.fixAiRunId;
-    } catch (error) {
-      logger.error(
-        `AI Insights: fix routing threw for insight ${insight.id?.toString()} (treating as no fix opened): ${error}`,
-      );
+    if (insight.insightType === AIInsightType.TraceLatencyRegression) {
+      try {
+        const fixResult: InsightFixRoutingResult =
+          await InsightFixRouting.routeInsightFix({
+            insight: insight,
+            project: project,
+          });
+        fixAiRunId = fixResult.fixAiRunId;
+      } catch (error) {
+        logger.error(
+          `AI Insights: fix routing threw for latency insight ${insight.id?.toString()} (treating as no fix opened): ${error}`,
+        );
+      }
     }
 
     if (fixAiRunId) {