@oneuptime/common 11.5.12 → 11.6.0

package/Server/Utils/AI/SRE/Insights/InsightTriageRunner.ts

@@ -1,13 +1,23 @@
 import ObjectID from "../../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../../Types/Date";
 import AIInsight from "../../../../../Models/DatabaseModels/AIInsight";
+import Project from "../../../../../Models/DatabaseModels/Project";
 import AIInsightEvidence from "../../../../../Types/AI/AIInsightEvidence";
+import AIInsightStatus, {
+  AIInsightStatusHelper,
+} from "../../../../../Types/AI/AIInsightStatus";
+import AIInsightType from "../../../../../Types/AI/AIInsightType";
+import ExceptionAIClassification from "../../../../../Types/AI/ExceptionAIClassification";
 import AIInsightService from "../../../../Services/AIInsightService";
+import ProjectService from "../../../../Services/ProjectService";
+import TelemetryExceptionService from "../../../../Services/TelemetryExceptionService";
 import { AI_INSIGHT_TRIAGE_FEATURE } from "../../../../Services/AIService";
 import AIInvestigationEngine from "../AIInvestigationEngine";
 import AIInvestigationQueue from "../InvestigationQueue";
+import InsightFixRouting, { InsightFixRoutingResult } from "./FixRouting";
 import { ConfidenceSignal } from "../ConfidenceSignal";
 import { ObservabilityAssistantResult } from "../../Chat/ObservabilityAssistant";
+import ToolResultSerializer from "../../Toolbox/Serializer";
 import logger from "../../../Logger";
 import CaptureSpan from "../../../Telemetry/CaptureSpan";
 
@@ -44,7 +54,17 @@ const PREVENTIVE_TRIAGE_FRAMING: string = `IMPORTANT — this is PREVENTIVE TRIA
 - Establish the most likely root cause of the finding.
 - Estimate the blast radius: which services/users/operations are affected, and how badly.
 - Recommend exactly ONE next action (the single most useful thing an engineer should do first).
-Cite the evidence for every factual claim. If the telemetry cannot support a conclusion, say so plainly. Do not refer to "the incident" — there is none.`;
+Cite the evidence for every factual claim. If the telemetry cannot support a conclusion, say so plainly. Do not refer to "the incident" — there is none.
+
+CLASSIFICATION — you MUST end your analysis with one line in EXACTLY this format, on its own line, with nothing after it:
+Classification: <verdict>
+where <verdict> is exactly one of: code-fault, user-error, expected-denial, infrastructure, unknown.
+- code-fault: a defect in the monitored code that a code change should fix.
+- user-error: an expected consequence of invalid end-user input (bad parameters, malformed values, garbage in a URL). The code rejected it correctly; the input was wrong.
+- expected-denial: an intentional check doing its job — authentication/authorization failures, plan or paywall rejections, rate limits, or security scanners/fuzzers tripping validation on purpose-built probes.
+- infrastructure: an environmental condition (network timeout, connection reset, resource exhaustion, dependency outage) rather than a logic defect.
+- unknown: the evidence does not support a confident verdict.
+This verdict gates automation: ONLY code-fault findings may get an automatic fix pull request, so classify conservatively — when torn between code-fault and anything else, pick the other one.`;
 
 export default class InsightTriageRunner {
   /*
@@ -63,11 +83,13 @@ export default class InsightTriageRunner {
     const { aiRunId, projectId, sentinelInsightId, attemptCount } = data;
 
     let contextSummary: string;
+    let insight: AIInsight | null = null;
     try {
-      const insight: AIInsight | null = await AIInsightService.findOneById({
+      insight = await AIInsightService.findOneById({
         id: sentinelInsightId,
         select: {
           _id: true,
+          projectId: true,
           insightType: true,
           severity: true,
           title: true,
@@ -75,6 +97,7 @@ export default class InsightTriageRunner {
           evidence: true,
           serviceName: true,
           traceId: true,
+          telemetryExceptionId: true,
           metricName: true,
           firstSeenAt: true,
           lastSeenAt: true,
@@ -118,30 +141,217 @@ export default class InsightTriageRunner {
           confidence: ConfidenceSignal;
           result: ObservabilityAssistantResult;
         }): Promise<void> => {
+          const classification: ExceptionAIClassification =
+            this.parseClassification(postData.analysisMarkdown);
+
           /*
-           * The quiet inbox — deliberately the ONLY write: no feed items,
-           * no workspace notification, no owner/on-call ping, no metrics,
-           * and no instrumentation-task trigger. Insights never page and
-           * never enter the notification pipeline (Preventive-lane rule);
-           * the summary waits on the Insights page until a human looks.
+           * The quiet inbox — the summary and verdict land on the insight
+           * row: no feed items, no workspace notification, no owner/on-call
+           * ping, no metrics, and no instrumentation-task trigger. Insights
+           * never page and never enter the notification pipeline
+           * (Preventive-lane rule); the summary waits on the Insights page
+           * until a human looks.
            */
           await AIInsightService.updateOneById({
             id: sentinelInsightId,
             data: {
               triageSummaryMarkdown: postData.analysisMarkdown,
               triageCompletedAt: OneUptimeDate.getCurrentDate(),
+              classification: classification,
             },
             props: { isRoot: true },
           });
 
           logger.debug(
-            `AI insights: triage summary posted for insight ${sentinelInsightId.toString()} (run ${aiRunId.toString()}, confident=${postData.confidence.confident} via ${postData.confidence.source}).`,
+            `AI insights: triage summary posted for insight ${sentinelInsightId.toString()} (run ${aiRunId.toString()}, classification=${classification}, confident=${postData.confidence.confident} via ${postData.confidence.source}).`,
           );
+
+          /*
+           * Verdict-driven follow-through. Best-effort by design: the
+           * triage summary is already posted, and a failure here must not
+           * fail the triage run itself.
+           */
+          try {
+            await this.actOnClassification({
+              insight: insight!,
+              classification: classification,
+            });
+          } catch (error) {
+            logger.error(
+              `AI insights: post-triage action failed for insight ${sentinelInsightId.toString()} (verdict ${classification}): ${error}`,
+            );
+          }
         },
       },
     });
   }
 
+  /*
+   * Parse the mandatory trailing "Classification: <verdict>" line out of
+   * the triage analysis. The LAST match wins (the model may discuss the
+   * taxonomy earlier in its reasoning). Anything unparseable is Unknown —
+   * and Unknown never routes a fix, so a malformed answer fails closed.
+   */
+  public static parseClassification(
+    analysisMarkdown: string,
+  ): ExceptionAIClassification {
+    const matches: Array<RegExpMatchArray> = Array.from(
+      (analysisMarkdown || "").matchAll(
+        /^\s*\**\s*classification\s*\**\s*[:=]\s*\**\s*(code-fault|user-error|expected-denial|infrastructure|unknown)\b/gim,
+      ),
+    );
+
+    const last: RegExpMatchArray | undefined = matches[matches.length - 1];
+
+    if (!last || !last[1]) {
+      return ExceptionAIClassification.Unknown;
+    }
+
+    return last[1].toLowerCase() as ExceptionAIClassification;
+  }
+
+  /*
+   * Act on the triage verdict:
+   *
+   * 1. Stamp the verdict onto the exception group row (aiClassification)
+   *    so the exceptions list can filter on it and the fix lane can skip
+   *    known-non-defects without re-triaging.
+   * 2. code-fault → route the automatic fix (InsightFixRouting owns every
+   *    gate: enableAi, enableInsightFixTasks, budget, readiness, dedupe)
+   *    and flip the insight to FixOpened when a run was queued.
+   * 3. expected-denial → optionally auto-archive the exception group when
+   *    the project opted in (autoArchiveNonActionableExceptions). Only
+   *    expected denials are archived — user errors and infrastructure
+   *    conditions stay visible for a human to judge.
+   */
+  private static async actOnClassification(data: {
+    insight: AIInsight;
+    classification: ExceptionAIClassification;
+  }): Promise<void> {
+    const { insight, classification } = data;
+
+    if (!insight.id || !insight.projectId) {
+      return;
+    }
+
+    /*
+     * The verdict stamp on the exception group is metadata and always
+     * lands; the AUTOMATION below (fix PR, auto-archive) additionally
+     * requires that no human closed the insight while the triage run sat
+     * in the queue or executed. The `insight` object is a snapshot from
+     * claim time — minutes to half an hour old — so the status MUST be
+     * re-read here: an insight a human Dismissed or Resolved in the
+     * meantime gets no PR and keeps its terminal status.
+     */
+    if (insight.telemetryExceptionId) {
+      await TelemetryExceptionService.updateOneById({
+        id: insight.telemetryExceptionId,
+        data: {
+          aiClassification: classification,
+        },
+        props: { isRoot: true },
+      });
+    }
+
+    const currentInsight: AIInsight | null = await AIInsightService.findOneById(
+      {
+        id: insight.id,
+        select: {
+          _id: true,
+          status: true,
+        },
+        props: { isRoot: true },
+      },
+    );
+
+    if (
+      !currentInsight ||
+      (currentInsight.status &&
+        AIInsightStatusHelper.isTerminalStatus(currentInsight.status))
+    ) {
+      logger.debug(
+        `AI insights: skipping post-triage automation for insight ${insight.id.toString()} — insight is ${currentInsight?.status || "deleted"}.`,
+      );
+      return;
+    }
+
+    const project: Project | null = await ProjectService.findOneById({
+      id: insight.projectId,
+      select: {
+        _id: true,
+        enableAi: true,
+        enableInsightFixTasks: true,
+        autoArchiveNonActionableExceptions: true,
+      },
+      props: { isRoot: true },
+    });
+
+    if (!project) {
+      return;
+    }
+
+    /*
+     * TraceLatencyRegression insights are fix-routed DETERMINISTICALLY at
+     * scan time (their span-tree evidence needs no LLM verdict — see
+     * InsightScanner.routeNewInsight); routing them again here would
+     * duplicate the run/PR. The verdict gate applies to exception
+     * insights only.
+     */
+    const isVerdictGatedInsight: boolean =
+      insight.insightType !== AIInsightType.TraceLatencyRegression;
+
+    if (
+      classification === ExceptionAIClassification.CodeFault &&
+      isVerdictGatedInsight
+    ) {
+      const fixResult: InsightFixRoutingResult =
+        await InsightFixRouting.routeInsightFix({
+          insight: insight,
+          project: project,
+        });
+
+      if (fixResult.fixAiRunId) {
+        /*
+         * Conditional flip: only a still-ActionRequired insight moves to
+         * FixOpened, so a dismissal landing between the status re-read
+         * above and this write can never be clobbered.
+         */
+        await AIInsightService.updateOneBy({
+          query: {
+            _id: insight.id.toString(),
+            status: AIInsightStatus.ActionRequired,
+          },
+          data: {
+            status: AIInsightStatus.FixOpened,
+            fixAiRunId: fixResult.fixAiRunId,
+          },
+          props: { isRoot: true },
+        });
+      }
+
+      return;
+    }
+
+    if (
+      classification === ExceptionAIClassification.ExpectedDenial &&
+      project.autoArchiveNonActionableExceptions === true &&
+      insight.telemetryExceptionId
+    ) {
+      await TelemetryExceptionService.updateOneById({
+        id: insight.telemetryExceptionId,
+        data: {
+          isArchived: true,
+          markedAsArchivedAt: OneUptimeDate.getCurrentDate(),
+        },
+        props: { isRoot: true },
+      });
+
+      logger.debug(
+        `AI insights: auto-archived exception ${insight.telemetryExceptionId.toString()} (expected-denial verdict, project opted in).`,
+      );
+    }
+  }
+
   // Build the compact insight record that seeds the triage.
   private static buildInsightSummary(insight: AIInsight): string {
     const lines: Array<string> = [];
@@ -191,7 +401,13 @@ export default class InsightTriageRunner {
       lines.push(renderedEvidence);
     }
 
-    return lines.join("\n");
+    /*
+     * Detector detailMarkdown and evidence quote raw exception messages —
+     * sweep secrets/PII before the text reaches the LLM provider. (The
+     * chat/investigation toolbox path already redacts its tool results;
+     * this closes the same gap for the detector-supplied context.)
+     */
+    return ToolResultSerializer.redact(lines.join("\n")).text;
   }
 
   /*

package/Server/Utils/AI/SRE/SubjectCodeFixRun.ts

@@ -116,6 +116,45 @@ export default class SubjectCodeFixRun {
     );
   }
 
+  /*
+   * Per-(service, recipe) dedupe for the service-scoped instrumentation
+   * recipes (ImproveLogging / ImproveTracing): at most one non-terminal
+   * run of a recipe per (project, telemetryServiceId). Same in-memory
+   * taskContext matching as the per-trace guard above — the id lives in
+   * JSON the Query layer cannot filter on, and the candidate set is
+   * bounded and rare.
+   */
+  @CaptureSpan()
+  public static async findNonTerminalRunForTelemetryService(data: {
+    projectId: ObjectID;
+    taskType: CodeFixTaskType;
+    telemetryServiceId: string;
+  }): Promise<AIRun | null> {
+    const activeRuns: Array<AIRun> = await AIRunService.findBy({
+      query: {
+        projectId: data.projectId,
+        runType: AIRunType.CodeFix,
+        codeFixTaskType: data.taskType,
+        status: QueryHelper.notIn([
+          AIRunStatus.Completed,
+          AIRunStatus.Error,
+          AIRunStatus.Cancelled,
+          AIRunStatus.Stale,
+        ]),
+      },
+      select: { _id: true, taskContext: true },
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    return (
+      activeRuns.find((run: AIRun): boolean => {
+        return run.taskContext?.telemetryServiceId === data.telemetryServiceId;
+      }) || null
+    );
+  }
+
   /*
    * Record the durable intent as a Queued CodeFix AIRun that the external
    * agent worker claims via /ai-agent-task/get-pending-task. `userId` is

package/Server/Utils/AI/SRE/TelemetryImprovementTaskTrigger.ts

@@ -0,0 +1,114 @@
+import ObjectID from "../../../../Types/ObjectID";
+import CodeFixTaskType from "../../../../Types/AI/CodeFixTaskType";
+import CodeFixTaskContext from "../../../../Types/AI/CodeFixTaskContext";
+import BadDataException from "../../../../Types/Exception/BadDataException";
+import AIRun from "../../../../Models/DatabaseModels/AIRun";
+import Service from "../../../../Models/DatabaseModels/Service";
+import ServiceService from "../../../Services/ServiceService";
+import SubjectCodeFixRun from "./SubjectCodeFixRun";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * AI SRE — the service-scoped instrumentation-improvement triggers:
+ * ImproveLogging and ImproveTracing. One click on a service's Logs or
+ * Traces page opens a draft PR that improves that service's logging or
+ * tracing hygiene — parameterized log messages, correct severities, spans
+ * on uninstrumented paths, proper exception recording — WITHOUT changing
+ * behavior.
+ *
+ * Like FixFromIncident and FixPerformance these are human-triggered from a
+ * user-facing endpoint (POST /ai-investigation/
+ * create-telemetry-improvement-task), so there is no project opt-in flag —
+ * the human in the loop IS the gate — and every unmet gate FAILS EARLY
+ * with a clear message. The recipe's grounding is the SERVICE: repository
+ * resolution at task-details time uses the service name (name-match /
+ * only-repository), the same path the incident-subject recipes use.
+ */
+export default class TelemetryImprovementTaskTrigger {
+  /*
+   * Gate and enqueue an ImproveLogging / ImproveTracing CodeFix run for a
+   * telemetry service. The caller must already have loaded the service
+   * under the USER's permissions (the access check) — this method reads
+   * and writes as root.
+   *
+   * Throws BadDataException naming the failed gate: unsupported recipe, no
+   * GitHub-App repository, a duplicate active run for the same service, or
+   * (via enqueueSubjectCodeFixRun) the daily fix-run budget.
+   */
+  @CaptureSpan()
+  public static async createTelemetryImprovementTask(data: {
+    projectId: ObjectID;
+    telemetryServiceId: ObjectID;
+    taskType: CodeFixTaskType;
+    userId: ObjectID;
+  }): Promise<AIRun> {
+    if (
+      data.taskType !== CodeFixTaskType.ImproveLogging &&
+      data.taskType !== CodeFixTaskType.ImproveTracing
+    ) {
+      throw new BadDataException(
+        `Task type "${data.taskType}" is not a telemetry-improvement recipe. Supported: ${CodeFixTaskType.ImproveLogging}, ${CodeFixTaskType.ImproveTracing}.`,
+      );
+    }
+
+    const service: Service | null = await ServiceService.findOneById({
+      id: data.telemetryServiceId,
+      select: {
+        _id: true,
+        name: true,
+        projectId: true,
+      },
+      props: { isRoot: true },
+    });
+
+    if (
+      !service ||
+      service.projectId?.toString() !== data.projectId.toString()
+    ) {
+      throw new BadDataException("Telemetry service not found.");
+    }
+
+    // Gate — a repository the agent can actually open a PR against.
+    const hasConnectedRepository: boolean =
+      await SubjectCodeFixRun.hasGitHubAppConnectedRepository(data.projectId);
+
+    if (!hasConnectedRepository) {
+      throw new BadDataException(
+        "No GitHub-App-connected repository exists for this project, so the agent has nowhere to open the pull request. Connect one under AI > Code Repositories.",
+      );
+    }
+
+    /*
+     * Gate — per-(service, recipe) dedupe: repeated clicks must not fan
+     * out into duplicate PRs.
+     */
+    const existingRun: AIRun | null =
+      await SubjectCodeFixRun.findNonTerminalRunForTelemetryService({
+        projectId: data.projectId,
+        taskType: data.taskType,
+        telemetryServiceId: data.telemetryServiceId.toString(),
+      });
+
+    if (existingRun) {
+      throw new BadDataException(
+        `An ${
+          data.taskType === CodeFixTaskType.ImproveLogging
+            ? "improve-logging"
+            : "improve-tracing"
+        } task is already queued or running for this service. Track its progress on the AI > Tasks page.`,
+      );
+    }
+
+    const taskContext: CodeFixTaskContext = {
+      telemetryServiceId: data.telemetryServiceId.toString(),
+      serviceName: service.name || undefined,
+    };
+
+    return SubjectCodeFixRun.enqueueSubjectCodeFixRun({
+      projectId: data.projectId,
+      taskType: data.taskType,
+      userId: data.userId,
+      taskContext,
+    });
+  }
+}

package/Server/Utils/AnalyticsDatabase/InsertDedupContext.ts

@@ -0,0 +1,59 @@
+import { AsyncLocalStorage } from "async_hooks";
+
+/**
+ * Ambient context that makes ClickHouse inserts idempotent across queue
+ * retries. The telemetry queue worker wraps each job in
+ * `runWithInsertDedup(jobId, ...)`; token consumers then derive
+ * `insert_deduplication_token = "<tokenBase>:<table>:<chunkIndex>"`, so a
+ * stalled-job recovery or attempts-based retry that re-processes the same
+ * payload re-issues byte-identical tokens and ClickHouse drops the duplicate
+ * blocks (on replicated tables; on plain MergeTree the token is ignored
+ * unless non_replicated_deduplication_window is set — no harm either way).
+ * The chunk counter is per table because one job inserts into several tables
+ * (e.g. Span + ExceptionInstance) in a deterministic order.
+ *
+ * Two consumers exist:
+ * - AnalyticsDatabaseService.insertJsonRows stamps direct inserts that run
+ *   inside the context without an explicit token.
+ * - TelemetryFanInWriter captures a token per submission AT SUBMIT TIME
+ *   (still inside the job's context) so cross-job batching downstream can
+ *   preserve per-job retry idempotence.
+ */
+export interface InsertDedupContextStore {
+  tokenBase: string;
+  chunkIndexByTable: Map<string, number>;
+}
+
+const insertDedupContext: AsyncLocalStorage<InsertDedupContextStore> =
+  new AsyncLocalStorage<InsertDedupContextStore>();
+
+export function runWithInsertDedup<T>(
+  tokenBase: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  return insertDedupContext.run(
+    { tokenBase, chunkIndexByTable: new Map<string, number>() },
+    fn,
+  );
+}
+
+/**
+ * Consume the next deterministic dedup token for `tableName` from the
+ * ambient context, or undefined when called outside a runWithInsertDedup
+ * scope. Consuming advances the per-table chunk counter, so calls must
+ * happen in a deterministic order relative to the payload — true for the
+ * ingest services, whose chunking is a pure function of the (byte-identical
+ * across retries) job body.
+ */
+export function nextInsertDedupToken(tableName: string): string | undefined {
+  const store: InsertDedupContextStore | undefined =
+    insertDedupContext.getStore();
+
+  if (!store) {
+    return undefined;
+  }
+
+  const chunkIndex: number = store.chunkIndexByTable.get(tableName) ?? 0;
+  store.chunkIndexByTable.set(tableName, chunkIndex + 1);
+  return `${store.tokenBase}:${tableName}:${chunkIndex}`;
+}

package/Server/Utils/Express.ts

@@ -103,6 +103,19 @@ class Express {
 
     if (!this.httpServer) {
       this.httpServer = createServer(this.app);
+
+      /*
+       * Keep upstream (nginx) connection reuse safe. When the ingress pools
+       * idle upstream connections (proxy keepalive), the backend must NOT be
+       * the side that closes an idle socket first — otherwise nginx can pick a
+       * socket the backend already closed and 502 a request it cannot safely
+       * retry (e.g. a POST on the telemetry ingest path). Node's default
+       * keepAliveTimeout (~5s) is well below nginx's upstream keepalive_timeout
+       * (60s), so raise it above that window. headersTimeout must stay greater
+       * than keepAliveTimeout or Node can cut an idle socket mid-request.
+       */
+      this.httpServer.keepAliveTimeout = 75 * 1000;
+      this.httpServer.headersTimeout = 76 * 1000;
     }
 
     /*