@oneuptime/common 11.4.0 → 11.5.0

package/Server/Utils/AI/Sentinel/SentinelInvestigationEngine.ts

@@ -5,10 +5,8 @@ import {
   AIChatCitation,
   AIRunEventResultSummary,
 } from "../../../../Types/AI/AIChatTypes";
-import AIRunType from "../../../../Types/AI/AIRunType";
 import AIRunStatus from "../../../../Types/AI/AIRunStatus";
 import AIRunEventType from "../../../../Types/AI/AIRunEventType";
-import AIRun from "../../../../Models/DatabaseModels/AIRun";
 import AIRunEvent from "../../../../Models/DatabaseModels/AIRunEvent";
 import Project from "../../../../Models/DatabaseModels/Project";
 import LlmProvider from "../../../../Models/DatabaseModels/LlmProvider";
@@ -16,6 +14,7 @@ import AIRunService from "../../../Services/AIRunService";
 import AIRunEventService from "../../../Services/AIRunEventService";
 import ProjectService from "../../../Services/ProjectService";
 import LlmProviderService from "../../../Services/LlmProviderService";
+import SentinelInvestigationQueue from "./InvestigationQueue";
 import ObservabilityAssistant, {
   ObservabilityAssistantResult,
   ObservabilityAssistantStep,
@@ -53,6 +52,14 @@ const MAX_OUTPUT_TOKENS: number = 2000;
  */
 const INCONCLUSIVE_RE: RegExp = /inconclusive[^.\n]*insufficient signal/i;
 
+/*
+ * Failures a retry cannot fix within the run's usefulness window: missing/
+ * broken provider configuration and budget exhaustion (both messages minted
+ * by our own gating in AIService/LLMService, so they are stable to match).
+ */
+const PERMANENT_FAILURE_RE: RegExp =
+  /no llm provider configured|llm provider type is not configured|token budget exhausted/i;
+
 // Maps a live agent step to the AIRunEvent type persisted for the glass-box trail.
 const STEP_EVENT_TYPE: Record<ObservabilityAssistantStepType, AIRunEventType> =
   {
@@ -67,7 +74,7 @@ const INVESTIGATION_PERSONA: string = `You are "Sentinel", OneUptime's autonomou
 
 Investigate like a senior on-call engineer:
 - Start from the affected monitors/services named in the signal.
-- Use your read tools to inspect the telemetry AROUND the signal's creation time: recent exceptions and their trends, error/latency metrics versus their normal range, failing traces, relevant logs, and recent changes / deploys.
+- Use your read tools to inspect the telemetry AROUND the signal's creation time: recent exceptions and their trends, error/latency metrics versus their normal range (use baseline_anomaly to judge a metric against its learned hour-of-week baseline quantitatively instead of eyeballing), failing traces, relevant logs, and recent changes / deploys.
 - Form the single most likely root-cause hypothesis. If the evidence is inconclusive, say so plainly and list what you checked — do NOT guess a cause the data does not support.
 - If the context lists past resolved incidents, check whether this is a RECURRENCE. If the current signal matches one, say so explicitly, reference that incident number, and note how it was resolved before — but still verify against the current telemetry.
 
@@ -83,17 +90,10 @@ Keep it tight and skimmable. You are read-only: never claim to have changed anyt
 export type SentinelSubjectType = "Incident" | "Alert";
 
 export interface InvestigationRequest {
-  projectId: ObjectID;
   // Label recorded on LlmLog, e.g. "Sentinel Incident Investigation".
   feature: string;
   // A compact markdown summary of the subject that seeds the investigation.
   contextSummary: string;
-  /*
-   * The subject that triggered this run — links the AIRun so the live panel can
-   * find "the investigation for this incident/alert". Exactly one is set.
-   */
-  subjectIncidentId?: ObjectID | undefined;
-  subjectAlertId?: ObjectID | undefined;
   /*
    * Called with the finished, branded, cited analysis so the caller can post it
    * to the subject's timeline. `isConfident` is false when Sentinel reported it
@@ -160,44 +160,38 @@ export default class SentinelInvestigationEngine {
   }
 
   /*
-   * Run one investigation end-to-end. Never throws — failures are logged and
-   * recorded on the AIRun.
+   * Execute an already-CLAIMED (Running) investigation run end-to-end.
+   * Called by SentinelInvestigationQueue after a successful CAS claim —
+   * cap/budget gating and run creation live in the queue now. Never throws;
+   * failures are handed to the queue's retry policy (failOrRequeue).
    */
   @CaptureSpan()
-  public static async investigate(
-    request: InvestigationRequest,
-  ): Promise<void> {
-    const { projectId } = request;
-
-    // Record the run up front so it is auditable even if it fails.
-    const run: AIRun = new AIRun();
-    run.projectId = projectId;
-    run.runType = AIRunType.Investigation;
-    run.status = AIRunStatus.Running;
-    run.startedAt = OneUptimeDate.getCurrentDate();
-    run.lastHeartbeatAt = OneUptimeDate.getCurrentDate();
-
-    if (request.subjectIncidentId) {
-      run.triggeredByIncidentId = request.subjectIncidentId;
-    }
-    if (request.subjectAlertId) {
-      run.triggeredByAlertId = request.subjectAlertId;
-    }
+  public static async executeRun(data: {
+    aiRunId: ObjectID;
+    projectId: ObjectID;
+    attemptCount: number;
+    request: InvestigationRequest;
+  }): Promise<void> {
+    const { aiRunId, projectId, request } = data;
 
-    let createdRun: AIRun;
+    /*
+     * Retried runs already have events from earlier attempts; continue the
+     * sequence so the glass-box trail stays ordered and shows every attempt.
+     */
+    let sequence: number = 0;
     try {
-      createdRun = await AIRunService.create({
-        data: run,
-        props: { isRoot: true },
-      });
+      sequence = (
+        await AIRunEventService.countBy({
+          query: { aiRunId },
+          props: { isRoot: true },
+        })
+      ).toNumber();
     } catch (error) {
-      logger.error(`Sentinel: failed to create investigation run: ${error}`);
-      return;
+      logger.error(
+        `Sentinel: failed to read event count for run ${aiRunId.toString()}; starting sequence at 0: ${error}`,
+      );
     }
 
-    const aiRunId: ObjectID = createdRun.id!;
-    let sequence: number = 0;
-
     await this.emitEvent({
       projectId,
       aiRunId,
@@ -234,10 +228,12 @@ export default class SentinelInvestigationEngine {
         citationId: step.citationId,
       });
 
-      // Keep the run visibly alive for the stale-run sweeper + live UI.
-      if (step.type === "llm_started") {
-        await this.touchHeartbeat(aiRunId);
-      }
+      /*
+       * Keep the run visibly alive for the stale-run sweeper + live UI on
+       * EVERY step — a slow self-hosted LLM call can approach the sweeper's
+       * timeout, so the heartbeat must be as frequent as we can make it.
+       */
+      await this.touchHeartbeat(aiRunId);
     };
 
     try {
@@ -256,18 +252,35 @@ export default class SentinelInvestigationEngine {
           onStep,
         });
 
-      await AIRunService.updateOneBy({
-        query: { _id: aiRunId.toString(), status: AIRunStatus.Running },
-        data: {
-          status: AIRunStatus.Completed,
-          completedAt: OneUptimeDate.getCurrentDate(),
-          lastHeartbeatAt: OneUptimeDate.getCurrentDate(),
-          llmCallCount: result.llmCallCount,
-          toolCallCount: result.toolCallCount,
-          totalTokens: result.totalTokens,
-        } as never,
-        props: { isRoot: true },
-      });
+      /*
+       * Atomic Running -> Completed. If we did NOT win this transition,
+       * another actor moved the run while we executed — most likely the
+       * stale sweeper falsely requeued it (slow LLM call outlasting the
+       * heartbeat window) and a second attempt is or will be running. In
+       * that case DO NOT post the analysis: the winning attempt will, and
+       * posting here would duplicate the RCA in the feed and workspace.
+       */
+      const completedCount: number = await AIRunService.attemptStatusTransition(
+        {
+          aiRunId,
+          fromStatus: AIRunStatus.Running,
+          set: {
+            status: AIRunStatus.Completed,
+            completedAt: OneUptimeDate.getCurrentDate(),
+            lastHeartbeatAt: OneUptimeDate.getCurrentDate(),
+            llmCallCount: result.llmCallCount,
+            toolCallCount: result.toolCallCount,
+            totalTokens: result.totalTokens,
+          },
+        },
+      );
+
+      if (completedCount === 0) {
+        logger.warn(
+          `Sentinel: run ${aiRunId.toString()} finished but was no longer Running (likely requeued as stale mid-flight); skipping postAnalysis to avoid a duplicate RCA.`,
+        );
+        return;
+      }
 
       await this.emitEvent({
         projectId,
@@ -300,16 +313,6 @@ export default class SentinelInvestigationEngine {
       const message: string =
         error instanceof Error ? error.message : String(error);
 
-      await AIRunService.updateOneBy({
-        query: { _id: aiRunId.toString(), status: AIRunStatus.Running },
-        data: {
-          status: AIRunStatus.Error,
-          completedAt: OneUptimeDate.getCurrentDate(),
-          errorMessage: message.substring(0, 480),
-        } as never,
-        props: { isRoot: true },
-      });
-
       await this.emitEvent({
         projectId,
         aiRunId,
@@ -317,8 +320,25 @@ export default class SentinelInvestigationEngine {
         eventType: AIRunEventType.RunFailed,
       });
 
+      /*
+       * Hand the failure to the queue's retry policy: transient errors
+       * requeue while attempts remain; permanent ones finalize as Error
+       * since retrying cannot help. Classification is by message, NOT by
+       * exception type: LLMService wraps transient provider failures (429s,
+       * 5xx, timeouts) in BadDataException too, so type-based classification
+       * would wrongly make the exact failures retries exist for permanent.
+       * Only configuration/budget gating — which a retry cannot change
+       * within the run's usefulness window — counts as permanent.
+       */
+      await SentinelInvestigationQueue.failOrRequeue({
+        aiRunId,
+        attemptCount: data.attemptCount,
+        errorMessage: message,
+        isPermanent: PERMANENT_FAILURE_RE.test(message),
+      });
+
       logger.error(
-        `Sentinel: investigation failed (run ${aiRunId.toString()}): ${message}`,
+        `Sentinel: investigation attempt ${data.attemptCount} failed (run ${aiRunId.toString()}): ${message}`,
       );
     }
   }

package/Server/Utils/AI/Toolbox/Index.ts

@@ -19,7 +19,7 @@ import { QueryMonitorsTool } from "./MonitorTools";
 import { TopExceptionsTool } from "./ExceptionTools";
 import { LogHistogramTool, SearchLogsTool } from "./LogTools";
 import { RecentChangesTool } from "./RecentChangesTools";
-import { QueryMetricsTool } from "./MetricTools";
+import { BaselineAnomalyTool, QueryMetricsTool } from "./MetricTools";
 import { GetTraceTool, QueryTracesTool } from "./TraceTools";
 import { LookupContextTool } from "./ContextTools";
 import {
@@ -67,6 +67,7 @@ export default class AIToolbox {
     SearchLogsTool,
     LogHistogramTool,
     QueryMetricsTool,
+    BaselineAnomalyTool,
     QueryTracesTool,
     GetTraceTool,
     RecentChangesTool,

package/Server/Utils/AI/Toolbox/MetricTools.ts

@@ -1,5 +1,7 @@
 import Metric from "../../../../Models/AnalyticsModels/Metric";
 import AggregationType from "../../../../Types/BaseDatabase/AggregationType";
+import DatabaseRequestType from "../../../Types/BaseDatabase/DatabaseRequestType";
+import ModelPermission from "../../../Types/AnalyticsDatabase/ModelPermission";
 import InBetween from "../../../../Types/BaseDatabase/InBetween";
 import { JSONObject } from "../../../../Types/JSON";
 import ObjectID from "../../../../Types/ObjectID";
@@ -11,6 +13,11 @@ import {
 } from "../../../../Types/AI/AIChatTypes";
 import AggregateBy from "../../../Types/AnalyticsDatabase/AggregateBy";
 import MetricService from "../../../Services/MetricService";
+import MetricBaselineService, {
+  BandPoint,
+  BaselineSummary,
+  MetricBaselineService as MetricBaselineServiceClass,
+} from "../../../Services/MetricBaselineService";
 import AggregatedResult from "../../../../Types/BaseDatabase/AggregatedResult";
 import AggregatedModel from "../../../../Types/BaseDatabase/AggregatedModel";
 import ToolResultSerializer, { SerializedResult } from "./Serializer";
@@ -177,3 +184,387 @@ export const QueryMetricsTool: ObservabilityTool = {
     };
   },
 };
+
+// The recent window whose average is judged against the baseline band.
+const BASELINE_CURRENT_WINDOW_MINUTES: number = 15;
+
+// Lookback rendered on the expected-range band chart.
+const BASELINE_CHART_HOURS: number = 6;
+
+export const BaselineAnomalyTool: ObservabilityTool = {
+  name: "baseline_anomaly",
+  description:
+    "Judge whether a metric is currently anomalous versus its learned hour-of-week baseline (rolling-window mean ± sigma band, the same math the anomaly monitors use). Answers 'is this value NORMAL for this metric at this time of week?' quantitatively — use it instead of eyeballing query_metrics output. Requires the exact metric name — discover names via lookup_context.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      metricName: {
+        type: "string",
+        description: "Exact metric name (required).",
+      },
+      entityId: {
+        type: "string",
+        description:
+          "Only data points from this service/host/monitor (its OneUptime ID). When omitted, the baseline spans every entity that ingests the metric.",
+      },
+      atTime: {
+        type: "string",
+        description:
+          "Evaluate at this ISO 8601 timestamp (e.g. the signal's creation time). Defaults to now.",
+      },
+      sensitivity: {
+        type: "string",
+        enum: ["Low", "Medium", "High"],
+        description:
+          "Band width: Low = 4 sigma (fewest anomalies), Medium = 3 sigma (default), High = 2 sigma.",
+      },
+      windowDays: {
+        type: "number",
+        enum: [...MetricBaselineServiceClass.WINDOW_DAYS_OPTIONS],
+        description: "Rolling baseline window in days (default 14).",
+      },
+    },
+    required: ["metricName"],
+  },
+  requiredPermissions: METRIC_READ_PERMISSIONS,
+  execute: async (
+    args: JSONObject,
+    ctx: ToolContext,
+  ): Promise<ToolExecutionResult> => {
+    const metricName: string | undefined = ToolArgs.getString(
+      args,
+      "metricName",
+    );
+
+    if (!metricName) {
+      throw new BadDataException(
+        "metricName is required. Use lookup_context with type 'metricNames' to discover metric names.",
+      );
+    }
+
+    const entityId: ObjectID | undefined = ToolArgs.getObjectID(
+      args,
+      "entityId",
+    );
+
+    let atTime: Date = OneUptimeDate.getCurrentDate();
+    const atTimeString: string | undefined = ToolArgs.getString(args, "atTime");
+    if (atTimeString) {
+      const parsed: Date = new Date(atTimeString);
+      if (isNaN(parsed.getTime())) {
+        throw new BadDataException(
+          `atTime "${atTimeString}" is not a valid ISO 8601 timestamp (e.g. 2024-01-31T14:00:00Z).`,
+        );
+      }
+      atTime = parsed;
+    }
+
+    const sensitivityString: string =
+      ToolArgs.getString(args, "sensitivity") || "Medium";
+    if (!["Low", "Medium", "High"].includes(sensitivityString)) {
+      throw new BadDataException(
+        `Invalid sensitivity: ${sensitivityString}. Must be Low, Medium or High.`,
+      );
+    }
+    const sensitivity: "Low" | "Medium" | "High" = sensitivityString as
+      | "Low"
+      | "Medium"
+      | "High";
+
+    const requestedWindowDays: number = ToolArgs.getNumber(args, "windowDays", {
+      defaultValue: MetricBaselineServiceClass.DEFAULT_WINDOW_DAYS,
+      min: MetricBaselineServiceClass.DEFAULT_WINDOW_DAYS,
+      max: MetricBaselineServiceClass.MAX_WINDOW_DAYS,
+    });
+    // Snap to the supported window options (they match the table's TTL).
+    const windowDays: number =
+      [...MetricBaselineServiceClass.WINDOW_DAYS_OPTIONS]
+        .reverse()
+        .find((option: number) => {
+          return option <= requestedWindowDays;
+        }) || MetricBaselineServiceClass.DEFAULT_WINDOW_DAYS;
+
+    /*
+     * The baseline queries are raw ClickHouse SQL scoped only by projectId —
+     * they skip the model layer's owned-scope filter. A label/owned-restricted
+     * user must therefore be pinned to one of THEIR services: project-wide
+     * baselines (no entityId) and other services' baselines are refused, the
+     * same posture as the other raw-SQL tools (see LogTools/TraceTools).
+     */
+    const accessibleServiceIds: Array<ObjectID> | null =
+      await ModelPermission.getAccessibleServiceIdsForAnalyticsModel(
+        Metric,
+        ctx.props,
+        DatabaseRequestType.Read,
+      );
+
+    if (accessibleServiceIds !== null) {
+      const isEntityAccessible: boolean = Boolean(
+        entityId &&
+          accessibleServiceIds.some((id: ObjectID) => {
+            return id.toString() === entityId.toString();
+          }),
+      );
+
+      if (!isEntityAccessible) {
+        throw new BadDataException(
+          "Your telemetry access is scoped to specific services, so a project-wide baseline is not available. Pass entityId set to the OneUptime ID of a service you can read (discover them via lookup_context).",
+        );
+      }
+    }
+
+    const hourOfWeek: number =
+      MetricBaselineServiceClass.computeHourOfWeek(atTime);
+
+    const baseline: BaselineSummary | null =
+      await MetricBaselineService.getBaseline({
+        projectId: ctx.projectId,
+        metricName,
+        primaryEntityId: entityId,
+        hourOfWeek,
+        windowDays,
+      });
+
+    const citationLabel: string = `baseline_anomaly(${metricName}${
+      entityId ? ` · ${entityId.toString()}` : ""
+    }) @ ${atTime.toISOString()}`;
+
+    /*
+     * Cold start: no baseline (or too few samples) for this hour-of-week cell.
+     * Report it explicitly — a missing baseline must read as "cannot judge",
+     * never as "not anomalous".
+     */
+    if (!baseline || !baseline.isReliable) {
+      const coverage: { totalSamples: number; oldestDay: Date | null } =
+        await MetricBaselineService.getCoverage({
+          projectId: ctx.projectId,
+          metricName,
+          primaryEntityId: entityId,
+        });
+
+      const verdictRow: JSONObject = {
+        verdict: "insufficient baseline data",
+        detail: `The metric has ${
+          baseline
+            ? `only ${baseline.sampleCount} sample(s)`
+            : "no baseline data"
+        } for this hour of week (needs at least ${MetricBaselineServiceClass.DEFAULT_MIN_SAMPLES}). Baseline coverage overall: ${coverage.totalSamples} total sample(s)${
+          coverage.oldestDay
+            ? `, oldest day ${OneUptimeDate.getDateAsFormattedString(coverage.oldestDay)}`
+            : ""
+        }. Do NOT treat this as "not anomalous" — the baseline simply cannot judge yet.`,
+        metricName,
+        entityId: entityId?.toString(),
+        evaluatedAt: atTime.toISOString(),
+        hourOfWeek,
+        windowDays,
+      };
+
+      const serializedColdStart: SerializedResult =
+        ToolResultSerializer.serializeRows([verdictRow]);
+
+      return {
+        dataForLlm: serializedColdStart.text,
+        rowCount: serializedColdStart.rowCount,
+        citationLabel,
+        citationTarget: { type: AIChatCitationTargetType.Metrics },
+        redactionCount: serializedColdStart.redactionCount,
+        isTruncated: serializedColdStart.isTruncated,
+      };
+    }
+
+    // Current value: average over the recent window ending at atTime.
+    const currentWindowStart: Date = OneUptimeDate.addRemoveMinutes(
+      atTime,
+      -1 * BASELINE_CURRENT_WINDOW_MINUTES,
+    );
+
+    const currentQuery: JSONObject = {
+      name: metricName,
+      time: new InBetween(currentWindowStart, atTime),
+    };
+    if (entityId) {
+      currentQuery["primaryEntityId"] = entityId;
+    }
+
+    const currentResult: AggregatedResult = await MetricService.aggregateBy({
+      aggregateColumnName: "value",
+      aggregationType: AggregationType.Avg,
+      aggregationTimestampColumnName: "time",
+      startTimestamp: currentWindowStart,
+      endTimestamp: atTime,
+      query: currentQuery as never,
+      limit: 200,
+      skip: 0,
+      props: ctx.props,
+    } as AggregateBy<Metric>);
+
+    const currentValues: Array<number> = currentResult.data
+      .map((item: AggregatedModel) => {
+        return item.value;
+      })
+      .filter((value: unknown): value is number => {
+        return typeof value === "number" && Number.isFinite(value);
+      });
+
+    const sigma: number =
+      MetricBaselineServiceClass.sigmaForSensitivity(sensitivity);
+    const expectedLow: number = baseline.mean - sigma * baseline.stddev;
+    const expectedHigh: number = baseline.mean + sigma * baseline.stddev;
+
+    let verdict: string;
+    let currentValue: number | null = null;
+    let zScore: number | null = null;
+
+    if (currentValues.length === 0) {
+      verdict = `no data points in the last ${BASELINE_CURRENT_WINDOW_MINUTES} minutes — the metric may have STOPPED reporting, which can itself be the anomaly`;
+    } else {
+      currentValue =
+        currentValues.reduce((sum: number, value: number) => {
+          return sum + value;
+        }, 0) / currentValues.length;
+
+      if (baseline.stddev <= 0) {
+        /*
+         * Zero-variance baseline: the band collapses to a point and
+         * floating-point noise alone would flag every sample. The anomaly
+         * monitors refuse to classify here (MetricMonitorCriteria) — match
+         * them rather than emitting guaranteed-false ANOMALOUS verdicts.
+         */
+        verdict = `cannot judge — the baseline has zero variance (the metric has been constant at ${baseline.mean}), so any band would flag every sample; compare the current value to the baseline mean directly instead`;
+      } else {
+        zScore = (currentValue - baseline.mean) / baseline.stddev;
+
+        if (currentValue > expectedHigh) {
+          verdict = `ANOMALOUS — above the expected range for this hour of week`;
+        } else if (currentValue < expectedLow) {
+          verdict = `ANOMALOUS — below the expected range for this hour of week`;
+        } else {
+          verdict = `normal — within the expected range for this hour of week`;
+        }
+      }
+    }
+
+    const row: JSONObject = {
+      verdict,
+      metricName,
+      entityId: entityId?.toString(),
+      evaluatedAt: atTime.toISOString(),
+      currentValue:
+        currentValue !== null
+          ? `${currentValue} (avg over the last ${BASELINE_CURRENT_WINDOW_MINUTES} min)`
+          : null,
+      expectedRange: `${expectedLow} to ${expectedHigh} (mean ${baseline.mean} ± ${sigma}σ, σ=${baseline.stddev})`,
+      zScore,
+      baselineMedian: baseline.median,
+      baselineP95: baseline.p95,
+      baselineSampleCount: baseline.sampleCount,
+      baselineWindowDays: baseline.windowDays,
+      hourOfWeek,
+      sensitivity,
+    };
+
+    const serialized: SerializedResult = ToolResultSerializer.serializeRows([
+      row,
+    ]);
+
+    // Expected-range band + actual values over the recent past, for the panel.
+    const chartStart: Date = OneUptimeDate.addRemoveHours(
+      atTime,
+      -1 * BASELINE_CHART_HOURS,
+    );
+
+    let bandSeries: Array<BandPoint> = [];
+    let actualPoints: Array<AIChatWidgetPoint> = [];
+    try {
+      bandSeries = await MetricBaselineService.getBandSeries({
+        projectId: ctx.projectId,
+        metricName,
+        primaryEntityId: entityId,
+        startTime: chartStart,
+        endTime: atTime,
+        intervalMinutes: 15,
+        sigmaCount: sigma,
+        windowDays,
+      });
+
+      const chartQuery: JSONObject = {
+        name: metricName,
+        time: new InBetween(chartStart, atTime),
+      };
+      if (entityId) {
+        chartQuery["primaryEntityId"] = entityId;
+      }
+
+      const chartActualResult: AggregatedResult =
+        await MetricService.aggregateBy({
+          aggregateColumnName: "value",
+          aggregationType: AggregationType.Avg,
+          aggregationTimestampColumnName: "time",
+          startTimestamp: chartStart,
+          endTimestamp: atTime,
+          query: chartQuery as never,
+          limit: 200,
+          skip: 0,
+          props: ctx.props,
+        } as AggregateBy<Metric>);
+
+      actualPoints = chartActualResult.data.map((item: AggregatedModel) => {
+        return {
+          x: OneUptimeDate.toString(OneUptimeDate.fromString(item.timestamp)),
+          y: typeof item.value === "number" ? item.value : null,
+        };
+      });
+    } catch {
+      // The band chart is decorative — never fail the verdict for it.
+    }
+
+    const toPoints: (
+      pick: (point: BandPoint) => number,
+    ) => Array<AIChatWidgetPoint> = (
+      pick: (point: BandPoint) => number,
+    ): Array<AIChatWidgetPoint> => {
+      return bandSeries.map((point: BandPoint) => {
+        return {
+          x: OneUptimeDate.toString(point.time),
+          y: pick(point),
+        };
+      });
+    };
+
+    return {
+      dataForLlm: serialized.text,
+      rowCount: serialized.rowCount,
+      citationLabel,
+      citationTarget: { type: AIChatCitationTargetType.Metrics },
+      redactionCount: serialized.redactionCount,
+      isTruncated: serialized.isTruncated,
+      widget:
+        bandSeries.length > 0
+          ? WidgetBuilder.timeSeries({
+              title: `${metricName} vs expected range`,
+              description: `Hour-of-week baseline (±${sigma}σ, ${windowDays}-day window), ${chartStart.toISOString()} – ${atTime.toISOString()}`,
+              series: [
+                {
+                  name: "expected high",
+                  points: toPoints((point: BandPoint) => {
+                    return point.expectedHigh;
+                  }),
+                },
+                {
+                  name: "expected low",
+                  points: toPoints((point: BandPoint) => {
+                    return point.expectedLow;
+                  }),
+                },
+                ...(actualPoints.length > 0
+                  ? [{ name: "actual", points: actualPoints }]
+                  : []),
+              ],
+              valueLabel: metricName,
+              link: { type: AIChatCitationTargetType.Metrics },
+            })
+          : undefined,
+    };
+  },
+};