@oneuptime/common 12.0.5 → 12.0.6

package/Server/Infrastructure/Queue.ts

@@ -560,6 +560,32 @@ export default class Queue {
     return waitingCount + activeCount + delayedCount;
   }
 
+  /*
+   * Backlog = jobs still waiting for a worker (waiting + delayed), for
+   * autoscaling signals. ACTIVE jobs are deliberately excluded: they are
+   * already being served, so to a scaler they are capacity, not demand.
+   * Telemetry jobs in particular sit in the active state for the fan-in
+   * writer's full flush window (up to TELEMETRY_FANIN_MAX_WAIT_MS) while
+   * awaiting their ClickHouse ack, so a fleet scaled on an active-inclusive
+   * count converges on pod count = job rate x ack latency — every new pod
+   * just parks more jobs without shortening the flush window. Scaling on
+   * waiting + delayed only breaks that feedback loop. getQueueSize above
+   * keeps the active-inclusive sum for ingest backpressure checks, where
+   * in-flight work genuinely counts against capacity.
+   */
+  @CaptureSpan()
+  public static async getQueueBacklogSize(
+    queueName: QueueName,
+  ): Promise<number> {
+    const queue: BullQueue = this.getQueue(queueName);
+    const [waitingCount, delayedCount]: [number, number] = await Promise.all([
+      queue.getWaitingCount(),
+      queue.getDelayedCount(),
+    ]);
+
+    return waitingCount + delayedCount;
+  }
+
   @CaptureSpan()
   public static async getQueueStats(queueName: QueueName): Promise<{
     waiting: number;

package/Server/Infrastructure/QueueWorker.ts

@@ -16,6 +16,7 @@ import Telemetry, {
 } from "../Utils/Telemetry";
 import Redis from "./Redis";
 import GracefulShutdown, { ShutdownPriority } from "../Utils/GracefulShutdown";
+import logger from "../Utils/Logger";
 
 export default class QueueWorker {
   @CaptureSpan()
@@ -120,6 +121,22 @@ export default class QueueWorker {
         : {}),
     });
 
+    /*
+     * Always log job failures to the container log, independent of telemetry
+     * configuration. Error visibility used to ride on @CaptureSpan's
+     * exception path, but the decorator is a passthrough when no span
+     * exporter is installed — without this listener a job that throws on
+     * such a deployment lands in the Redis failed set with no log line.
+     */
+    worker.on("failed", (job: QueueJob | undefined, error: Error) => {
+      logger.error(
+        `Queue job failed: ${queueName}/${job?.name || "unknown"} (job ${
+          job?.id ?? "unknown"
+        })`,
+      );
+      logger.error(error);
+    });
+
     /*
      * Stop pulling new jobs and let in-flight ones finish on shutdown. Runs in
      * the Workers tier — before datastores are drained — so jobs mid-flight can

package/Server/Middleware/TelemetryIngest.ts

@@ -73,16 +73,21 @@ export default class TelemetryIngest {
         );
 
       if (!projectId) {
+        /*
+         * The token value itself is deliberately NOT logged: ingestion
+         * keys are secrets, and a mistyped-but-nearly-valid token in a
+         * log line is a credential leak. The request attributes give
+         * operators enough to correlate with the sender.
+         */
         logger.error(
-          "Invalid service token: " + oneuptimeToken,
+          "Invalid service token.",
           getLogAttributesFromRequest(req as any),
         );
 
         /*
          * 401 is deliberate (see the missing-token branch above): a
          * silent 200 drops the payload while the client believes the
-         * export succeeded. The token value is logged server-side but
-         * intentionally not echoed back in the response body.
+         * export succeeded.
          */
         return Response.sendErrorResponse(
           req,

package/Server/Services/AIInsightService.ts

@@ -8,7 +8,9 @@ import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import BadDataException from "../../Types/Exception/BadDataException";
-import AIInsightStatus from "../../Types/AI/AIInsightStatus";
+import AIInsightStatus, {
+  AIInsightStatusHelper,
+} from "../../Types/AI/AIInsightStatus";
 import AIInsightHumanVerdict from "../../Types/AI/AIInsightHumanVerdict";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 
@@ -25,6 +27,18 @@ export class Service extends DatabaseService<AIInsight> {
     super(AIInsight);
   }
 
+  /*
+   * The open state an insight returns to when a human takes a terminal
+   * action back: FixOpened when an AI fix task was queued for it (that run
+   * is still the live thread of work), else ActionRequired — the same two
+   * states the scanner routes a freshly detected insight to.
+   */
+  private getReopenStatus(insight: AIInsight): AIInsightStatus {
+    return insight.fixAiRunId
+      ? AIInsightStatus.FixOpened
+      : AIInsightStatus.ActionRequired;
+  }
+
   /*
    * Human verdict capture — this IS the G11 precision measurement:
    * confirm/dismiss rates per insight type tell us how precise each
@@ -32,34 +46,123 @@ export class Service extends DatabaseService<AIInsight> {
    *
    * Overwriting an existing verdict is deliberate — people change their
    * minds; the latest verdict wins (per-insight state, not an audit trail).
-   * Dismissed is a human terminal state, so it also closes the insight;
-   * Confirmed leaves the status untouched (the insight stays open until a
-   * human resolves it or a fix lands).
+   * Dismissed is a human terminal state, so it also closes the insight.
+   *
+   * Confirmed normally leaves the status untouched (the insight stays open
+   * until a human resolves it or a fix lands) — with one exception: an
+   * insight closed AS Dismissed is closed on the grounds that it was noise,
+   * which a Confirmed verdict directly contradicts. Leaving it closed there
+   * produced the "Dismissed / Confirmed" pill pair with no way back, and
+   * kept the InsightStore dismissal cooldown suppressing the very finding
+   * the human just called real. So confirming a dismissed insight reopens
+   * it, which is also what makes Confirm a working undo for a misclick.
    *
    * Callers must have already access-checked the insight under the USER's
    * permissions — insight rows are server-authored (empty update ACL), so
-   * this writes as root.
+   * this reads and writes as root.
    */
   @CaptureSpan()
   public async applyHumanVerdict(data: {
     insightId: ObjectID;
     verdict: AIInsightHumanVerdict;
     byUserId: ObjectID;
-  }): Promise<{ insightId: ObjectID; verdict: AIInsightHumanVerdict }> {
-    await this.updateOneById({
+  }): Promise<{
+    insightId: ObjectID;
+    verdict: AIInsightHumanVerdict;
+    status: AIInsightStatus | null;
+  }> {
+    const insight: AIInsight | null = await this.findOneById({
       id: data.insightId,
+      select: { _id: true, status: true, fixAiRunId: true },
+      props: { isRoot: true },
+    });
+
+    if (!insight || !insight.id) {
+      throw new BadDataException("AI insight not found.");
+    }
+
+    let nextStatus: AIInsightStatus | null = null;
+
+    if (data.verdict === AIInsightHumanVerdict.Dismissed) {
+      nextStatus = AIInsightStatus.Dismissed;
+    } else if (insight.status === AIInsightStatus.Dismissed) {
+      nextStatus = this.getReopenStatus(insight);
+    }
+
+    await this.updateOneById({
+      id: insight.id,
       data: {
         humanVerdict: data.verdict,
         humanVerdictAt: OneUptimeDate.getCurrentDate(),
         humanVerdictByUserId: data.byUserId,
-        ...(data.verdict === AIInsightHumanVerdict.Dismissed
-          ? { status: AIInsightStatus.Dismissed }
-          : {}),
+        ...(nextStatus ? { status: nextStatus } : {}),
+      },
+      props: { isRoot: true },
+    });
+
+    return {
+      insightId: insight.id,
+      verdict: data.verdict,
+      status: nextStatus || insight.status || null,
+    };
+  }
+
+  /*
+   * Undo a terminal action: put a Resolved or Dismissed insight back into
+   * the open queue and drop the human verdict that closed it.
+   *
+   * Both terminal states are one click away from a full stop — a dismissal
+   * additionally suppresses the same fingerprint for the whole InsightStore
+   * cooldown, so a misclick silently blinds the project to that finding for
+   * days. Reopening is the way back: the status returns to the open state
+   * the scanner would have routed to, and the verdict is CLEARED rather
+   * than kept, because a verdict the human has taken back must not keep
+   * counting toward the per-detector precision measurement (and a stale
+   * humanVerdictAt would otherwise anchor a future cooldown).
+   *
+   * Idempotent: reopening an already-open insight is a no-op that reports
+   * the current status, so a double click (or two people clicking at once)
+   * cannot rewrite a status somebody else just set.
+   *
+   * Callers must have already access-checked the insight under the USER's
+   * permissions — this reads and writes as root.
+   */
+  @CaptureSpan()
+  public async reopenInsight(data: {
+    insightId: ObjectID;
+  }): Promise<{ insightId: ObjectID; status: AIInsightStatus }> {
+    const insight: AIInsight | null = await this.findOneById({
+      id: data.insightId,
+      select: { _id: true, status: true, fixAiRunId: true },
+      props: { isRoot: true },
+    });
+
+    if (!insight || !insight.id) {
+      throw new BadDataException("AI insight not found.");
+    }
+
+    if (
+      insight.status &&
+      !AIInsightStatusHelper.isTerminalStatus(insight.status)
+    ) {
+      return { insightId: insight.id, status: insight.status };
+    }
+
+    const status: AIInsightStatus = this.getReopenStatus(insight);
+
+    await this.updateOneById({
+      id: insight.id,
+      data: {
+        status: status,
+        // Explicit nulls — the columns are cleared, not left as they were.
+        humanVerdict: null as unknown as AIInsightHumanVerdict,
+        humanVerdictAt: null as unknown as Date,
+        humanVerdictByUserId: null as unknown as ObjectID,
       },
       props: { isRoot: true },
     });
 
-    return { insightId: data.insightId, verdict: data.verdict };
+    return { insightId: insight.id, status: status };
   }
 
   /*
@@ -79,7 +182,7 @@ export class Service extends DatabaseService<AIInsight> {
   }): Promise<{ insightId: ObjectID; status: AIInsightStatus }> {
     const insight: AIInsight | null = await this.findOneById({
       id: data.insightId,
-      select: { _id: true, humanVerdict: true },
+      select: { _id: true, humanVerdict: true, status: true },
       props: { isRoot: true },
     });
 
@@ -87,6 +190,23 @@ export class Service extends DatabaseService<AIInsight> {
       throw new BadDataException("AI insight not found.");
     }
 
+    /*
+     * Closing an already-closed insight is a no-op the caller should know
+     * about — and resolving a DISMISSED one is actively harmful: the store's
+     * re-file cooldown is keyed on the current terminal status, so the flip
+     * would silently trade a dismissal's 7-day quiet for a resolve's 24-hour
+     * one and let a finding a human called noise come back. The dashboard
+     * hides both actions on terminal insights; this closes the API path.
+     */
+    if (
+      insight.status &&
+      AIInsightStatusHelper.isTerminalStatus(insight.status)
+    ) {
+      throw new BadDataException(
+        `This AI insight is already ${insight.status.toLowerCase()} and cannot be resolved again.`,
+      );
+    }
+
     await this.updateOneById({
       id: insight.id,
       data: {

package/Server/Services/AIRunService.ts

@@ -99,6 +99,40 @@ export class Service extends DatabaseService<Model> {
     return result.affected || 0;
   }
 
+  /*
+   * Persist the AI-written TL;DR of a completed investigation's analysis.
+   * Display-only, so this is a plain conditional UPDATE rather than a claim,
+   * but it is still write-once: scoped to the run, to the Completed state,
+   * and to a row that has no summary yet.
+   *
+   * The IS NULL guard is what makes the invariant structural rather than
+   * incidental. A stale attempt — one that lost the Completed CAS because the
+   * sweeper falsely requeued it — is kept out today only by statement order
+   * in the engine, and by then the WINNER has already written both the report
+   * and its summary, so a status-only guard would happily let the loser
+   * overwrite a summary that describes a different analysis. Returns the rows
+   * changed; 0 means the run moved on, was removed, or is already summarized.
+   */
+  @CaptureSpan()
+  public async setInvestigationAnalysisTldr(data: {
+    aiRunId: ObjectID;
+    analysisTldr: string;
+  }): Promise<number> {
+    const result: UpdateResult = await this.getRepository()
+      .createQueryBuilder()
+      .update(Model)
+      .set({
+        analysisTldr: data.analysisTldr,
+      } as QueryDeepPartialEntity<Model>)
+      .where('"_id" = :id', { id: data.aiRunId.toString() })
+      .andWhere('"status" = :status', { status: AIRunStatus.Completed })
+      .andWhere('"analysisTldr" IS NULL')
+      .andWhere('"deletedAt" IS NULL')
+      .execute();
+
+    return result.affected || 0;
+  }
+
   /*
    * Atomically settle the code-fix decision written as Pending by the
    * winning investigation completion transition. The recommendation and

package/Server/Services/AIService.ts

@@ -88,6 +88,15 @@ export const AI_INVESTIGATION_GRADING_FEATURE: string =
 export const AI_CONFIDENCE_CLASSIFICATION_FEATURE: string =
   "AI Confidence Classification";
 
+/*
+ * The LlmLog feature name for the investigation TL;DR (InvestigationTldr) —
+ * one constrained call per completed investigation that has an analysis,
+ * outside the per-run caps but inside this daily budget. A budget rejection
+ * (or any other failure) simply leaves the run without a TL;DR; the report
+ * is still published and the panel omits the block.
+ */
+export const AI_INVESTIGATION_TLDR_FEATURE: string = "AI Investigation TLDR";
+
 /*
  * The LlmLog feature name for server-mediated code-fix agent completions
  * (B4 Tier 0) — the in-house coding agent's tool loop routes every LLM call
@@ -196,6 +205,12 @@ export const AUTONOMOUS_AI_FEATURES: Array<string> = [
   AI_INVESTIGATION_GRADING_FEATURE,
   // One constrained call per completed investigation (ConfidenceSignal).
   AI_CONFIDENCE_CLASSIFICATION_FEATURE,
+  /*
+   * One constrained call per completed investigation that produced an
+   * analysis (InvestigationTldr) — same storm shape as the classification
+   * above, so the same daily budget must cover it.
+   */
+  AI_INVESTIGATION_TLDR_FEATURE,
   /*
    * Server-mediated code-fix agent completions (B4 Tier 0). Fix runs are
    * user-triggered, but the tool loop then runs unattended for up to ~40
@@ -563,9 +578,9 @@ export class Service extends BaseService {
 
       if (budget.exhausted) {
         const settingsLocation: string = request.incidentId
-          ? "Incidents > Settings > AI"
+          ? "Incidents > AI > Investigation"
           : request.alertId
-            ? "Alerts > Settings > AI"
+            ? "Alerts > AI > Investigation"
             : "Project Settings > AI > AI Guardrails";
         const budgetMessage: string = `Daily autonomous AI token budget exhausted (${budget.usedTokensToday.toLocaleString()} of ${budget.limitInTokens?.toLocaleString()} tokens used today). Autonomous AI requests resume tomorrow (UTC) — raise or unset the limit under ${settingsLocation}.`;
 

package/Server/Services/AlertService.ts

@@ -39,6 +39,7 @@ import { IsBillingEnabled } from "../EnvironmentConfig";
 import logger, { LogAttributes } from "../Utils/Logger";
 import Semaphore, { SemaphoreMutex } from "../Infrastructure/Semaphore";
 import TelemetryUtil from "../Utils/Telemetry/Telemetry";
+import MetricResourceAttributeUtil from "../../Utils/Metrics/MetricResourceAttributeUtil";
 import MutableMetricService from "./MutableMetricService";
 import GlobalConfigService from "./GlobalConfigService";
 import GlobalConfig from "../../Models/DatabaseModels/GlobalConfig";
@@ -1616,6 +1617,16 @@ ${alertSeverity.name}
           _id: true,
           name: true,
         },
+        /*
+         * A project's own taxonomy. Both become metric attributes below so
+         * dashboards can group MTTA/MTTR by the dimensions the project
+         * actually thinks in.
+         */
+        labels: {
+          _id: true,
+          name: true,
+        },
+        customFields: true,
       },
       props: {
         isRoot: true,
@@ -1692,6 +1703,29 @@ ${alertSeverity.name}
         metricRetentionDays,
       );
 
+      /*
+       * Dimensions shared by every alert metric. Built once so all four
+       * metrics below record the identical attribute set — a dashboard that
+       * groups by oneuptime.label.product must not find the dimension on
+       * AlertCount but missing from TimeToResolve.
+       */
+      const baseMetricAttributes: JSONObject = {
+        alertId: data.alertId.toString(),
+        projectId: alert.projectId.toString(),
+        monitorId: alert.monitor?._id?.toString(),
+        monitorName: alert.monitor?.name?.toString(),
+        alertSeverityId: alert.alertSeverity?._id?.toString(),
+        alertSeverityName: alert.alertSeverity?.name?.toString(),
+        /*
+         * oneuptime.label.* / oneuptime.customField.* — namespaced, so they can
+         * never collide with the unprefixed dimensions above.
+         */
+        ...MetricResourceAttributeUtil.getResourceAttributes({
+          labels: alert.labels,
+          customFields: alert.customFields,
+        }),
+      };
+
       // now we need to create new metrics for this alert - TimeToAcknowledge, TimeToResolve, AlertCount, AlertDuration
       const alertStartsAt: Date =
         firstAlertStateTimeline?.startsAt ||
@@ -1714,14 +1748,7 @@ ${alertSeverity.name}
       alertCountMetric.name = AlertMetricType.AlertCount;
       alertCountMetric.metricPointId = AlertMetricType.AlertCount;
       alertCountMetric.value = 1;
-      alertCountMetric.attributes = {
-        alertId: data.alertId.toString(),
-        projectId: alert.projectId.toString(),
-        monitorId: alert.monitor?._id?.toString(),
-        monitorName: alert.monitor?.name?.toString(),
-        alertSeverityId: alert.alertSeverity?._id?.toString(),
-        alertSeverityName: alert.alertSeverity?.name?.toString(),
-      };
+      alertCountMetric.attributes = { ...baseMetricAttributes };
       alertCountMetric.attributeKeys = TelemetryUtil.getAttributeKeys(
         alertCountMetric.attributes,
       );
@@ -1768,14 +1795,7 @@ ${alertSeverity.name}
             ackAlertStateTimeline?.startsAt || OneUptimeDate.getCurrentDate(),
             alertStartsAt,
           );
-          timeToAcknowledgeMetric.attributes = {
-            alertId: data.alertId.toString(),
-            projectId: alert.projectId.toString(),
-            monitorId: alert.monitor?._id?.toString(),
-            monitorName: alert.monitor?.name?.toString(),
-            alertSeverityId: alert.alertSeverity?._id?.toString(),
-            alertSeverityName: alert.alertSeverity?.name?.toString(),
-          };
+          timeToAcknowledgeMetric.attributes = { ...baseMetricAttributes };
           timeToAcknowledgeMetric.attributeKeys =
             TelemetryUtil.getAttributeKeys(timeToAcknowledgeMetric.attributes);
 
@@ -1825,14 +1845,7 @@ ${alertSeverity.name}
             OneUptimeDate.getCurrentDate(),
           alertStartsAt,
         );
-        timeToResolveMetric.attributes = {
-          alertId: data.alertId.toString(),
-          projectId: alert.projectId.toString(),
-          monitorId: alert.monitor?._id?.toString(),
-          monitorName: alert.monitor?.name?.toString(),
-          alertSeverityId: alert.alertSeverity?._id?.toString(),
-          alertSeverityName: alert.alertSeverity?.name?.toString(),
-        };
+        timeToResolveMetric.attributes = { ...baseMetricAttributes };
         timeToResolveMetric.attributeKeys = TelemetryUtil.getAttributeKeys(
           timeToResolveMetric.attributes,
         );
@@ -1872,14 +1885,7 @@ ${alertSeverity.name}
           alertEndsAt,
           alertStartsAt,
         );
-        alertDurationMetric.attributes = {
-          alertId: data.alertId.toString(),
-          projectId: alert.projectId.toString(),
-          monitorId: alert.monitor?._id?.toString(),
-          monitorName: alert.monitor?.name?.toString(),
-          alertSeverityId: alert.alertSeverity?._id?.toString(),
-          alertSeverityName: alert.alertSeverity?.name?.toString(),
-        };
+        alertDurationMetric.attributes = { ...baseMetricAttributes };
         alertDurationMetric.attributeKeys = TelemetryUtil.getAttributeKeys(
           alertDurationMetric.attributes,
         );

package/Server/Services/AnalyticsDatabaseService.ts

@@ -175,10 +175,14 @@ export const MigrationExecuteOptions: ClickhouseExecuteOptions = {
  * TelemetryFanInWriter can consume deterministic tokens without an import
  * cycle. Re-exported here for existing callers.
  *
- * HTTP-path inserts run outside the context and are always fire-and-forget
- * (wait_for_async_insert=0) — flush waiting, when opted into via
- * TELEMETRY_WAIT_FOR_ASYNC_INSERT, is only affordable off the request
- * thread and therefore only applies to tokened (queue-job) inserts.
+ * The ack mode is independent of tokening: TELEMETRY_WAIT_FOR_ASYNC_INSERT
+ * (shouldWaitForAsyncInsert below) is applied to EVERY insertJsonRows call,
+ * with or without a dedup token. Whether a telemetry queue job's writes
+ * carry per-job tokens from this context is a separate policy decided in
+ * the worker: the high-volume signals (traces/logs/metrics) run untokened
+ * so the fan-in writer can merge them into cross-job INSERTs under minted
+ * per-batch tokens — see shouldUseInsertDedup in
+ * App/FeatureSet/Telemetry/Jobs/TelemetryIngest/ProcessTelemetry.ts.
  */
 export {
   runWithInsertDedup,