@oneuptime/common 11.3.26 → 11.4.0

package/Server/Utils/AI/Sentinel/SentinelMemory.ts

@@ -0,0 +1,167 @@
+import ObjectID from "../../../../Types/ObjectID";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import Incident from "../../../../Models/DatabaseModels/Incident";
+import Monitor from "../../../../Models/DatabaseModels/Monitor";
+import IncidentService from "../../../Services/IncidentService";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+import logger from "../../Logger";
+
+/*
+ * Sentinel — episodic memory (Phase 4, "I've seen this before").
+ *
+ * The first, no-new-storage cut of the recurrence short-circuit: instead of a
+ * vector store, retrieve past RESOLVED incidents in the same project that share
+ * monitors/labels with the current signal and carry a recorded root cause, then
+ * feed a compact digest into the investigation so the agent can recognise a
+ * recurrence and reference the prior fix.
+ *
+ * The full episodic store (pgvector embeddings over postmortems/runbooks + a
+ * temporal knowledge graph) is the deferred, larger build — this delivers the
+ * user-visible "this matches #123" behaviour on data we already own.
+ */
+
+// How many recent incidents to consider, and how many prior cases to surface.
+const CANDIDATE_LIMIT: number = 30;
+const MAX_CASES: number = 4;
+const MAX_ROOT_CAUSE_CHARS: number = 300;
+
+const SHARED_MONITOR_SCORE: number = 2;
+const SHARED_LABEL_SCORE: number = 1;
+
+export default class SentinelMemory {
+  /*
+   * Returns a markdown "past resolved incidents" section to append to the
+   * investigation context, or an empty string when there is nothing useful.
+   * Never throws — memory is a nice-to-have, not load-bearing.
+   */
+  @CaptureSpan()
+  public static async getPriorSimilarIncidentsContext(data: {
+    projectId: ObjectID;
+    currentIncidentId: ObjectID;
+    monitorNames: Array<string>;
+    labelNames: Array<string>;
+  }): Promise<string> {
+    try {
+      const candidates: Array<Incident> = await IncidentService.findBy({
+        query: {
+          projectId: data.projectId,
+        },
+        select: {
+          _id: true,
+          incidentNumber: true,
+          title: true,
+          rootCause: true,
+          createdAt: true,
+          currentIncidentState: {
+            isResolvedState: true,
+          },
+          monitors: {
+            name: true,
+          },
+          labels: {
+            name: true,
+          },
+        },
+        sort: {
+          createdAt: SortOrder.Descending,
+        },
+        limit: CANDIDATE_LIMIT,
+        skip: 0,
+        props: { isRoot: true },
+      });
+
+      const currentIdString: string = data.currentIncidentId.toString();
+      const monitorSet: Set<string> = new Set(
+        data.monitorNames.map((n: string) => {
+          return n.toLowerCase();
+        }),
+      );
+      const labelSet: Set<string> = new Set(
+        data.labelNames.map((n: string) => {
+          return n.toLowerCase();
+        }),
+      );
+
+      const scored: Array<{ incident: Incident; score: number }> = candidates
+        .filter((incident: Incident) => {
+          return (
+            incident.id?.toString() !== currentIdString &&
+            incident.currentIncidentState?.isResolvedState === true &&
+            Boolean(incident.rootCause && incident.rootCause.trim().length > 0)
+          );
+        })
+        .map((incident: Incident) => {
+          let score: number = 0;
+
+          for (const monitor of incident.monitors || []) {
+            if (monitor.name && monitorSet.has(monitor.name.toLowerCase())) {
+              score += SHARED_MONITOR_SCORE;
+            }
+          }
+
+          for (const label of incident.labels || []) {
+            if (label.name && labelSet.has(label.name.toLowerCase())) {
+              score += SHARED_LABEL_SCORE;
+            }
+          }
+
+          return { incident, score };
+        });
+
+      /*
+       * Prefer cases that actually share a monitor/label; otherwise fall back to
+       * the most recent resolved incidents (V8's sort is stable, keeping recency).
+       */
+      const relevant: Array<{ incident: Incident; score: number }> = scored
+        .filter((item: { incident: Incident; score: number }) => {
+          return item.score > 0;
+        })
+        .sort(
+          (
+            a: { incident: Incident; score: number },
+            b: { incident: Incident; score: number },
+          ) => {
+            return b.score - a.score;
+          },
+        );
+
+      const chosen: Array<{ incident: Incident; score: number }> = (
+        relevant.length > 0 ? relevant : scored
+      ).slice(0, MAX_CASES);
+
+      if (chosen.length === 0) {
+        return "";
+      }
+
+      let markdown: string =
+        "\n\n# Past resolved incidents in this project (recurrence context)\n" +
+        "If the current signal matches one of these, this may be a recurrence — say so explicitly and reference the incident number and how it was resolved.\n";
+
+      for (const { incident } of chosen) {
+        const rootCause: string = (incident.rootCause || "").trim();
+        const shortRootCause: string =
+          rootCause.length > MAX_ROOT_CAUSE_CHARS
+            ? `${rootCause.substring(0, MAX_ROOT_CAUSE_CHARS)}…`
+            : rootCause;
+
+        markdown += `\n- **#${incident.incidentNumber} — ${incident.title || "Untitled"}**`;
+        markdown += `\n  Previously resolved. Root cause: ${shortRootCause}`;
+
+        const monitorNames: Array<string> = (incident.monitors || [])
+          .map((m: Monitor) => {
+            return m.name || "";
+          })
+          .filter(Boolean);
+
+        if (monitorNames.length > 0) {
+          markdown += `\n  Monitors: ${monitorNames.join(", ")}`;
+        }
+      }
+
+      return markdown;
+    } catch (error) {
+      logger.error(`Sentinel: failed to load recurrence context: ${error}`);
+      return "";
+    }
+  }
+}

package/Server/Utils/AI/Toolbox/IncidentTools.ts

@@ -35,7 +35,7 @@ const resolveReadPermissions: () => Array<Permission> =
 export const QueryIncidentsTool: ObservabilityTool = {
   name: "query_incidents",
   description:
-    "Query incidents in this project. Returns the most recent incidents with their current state and severity. Pass incidentId to get full details of one incident.",
+    "Query incidents in this project. Returns the most recent incidents with their current state and severity. Pass incidentId to get full details of one incident — including its affected monitors, labels, root cause, remediation and postmortem notes, and the incident window (telemetryWindowStart) to pivot into logs/traces/metrics for the affected services.",
   inputSchema: {
     type: "object",
     properties: {
@@ -81,10 +81,44 @@ export const QueryIncidentsTool: ObservabilityTool = {
           incidentSeverity: {
             name: true,
           },
+          monitors: {
+            _id: true,
+            name: true,
+          },
+          labels: {
+            name: true,
+          },
+          rootCause: true,
+          remediationNotes: true,
+          postmortemNote: true,
         },
         props: ctx.props,
       });
 
+      /*
+       * Surface the incident's blast radius (affected monitors + labels) and
+       * its own window so the model can pivot from an incident into the
+       * logs/traces/metrics of the affected services over the right time
+       * range, instead of guessing which service or hour to look at.
+       */
+      const affectedMonitors: string = (incident?.monitors || [])
+        .map((monitor: { name?: string }) => {
+          return monitor.name || "";
+        })
+        .filter((value: string) => {
+          return value.length > 0;
+        })
+        .join(", ");
+
+      const incidentLabels: string = (incident?.labels || [])
+        .map((label: { name?: string }) => {
+          return label.name || "";
+        })
+        .filter((value: string) => {
+          return value.length > 0;
+        })
+        .join(", ");
+
       const rows: Array<JSONObject> = incident
         ? [
             {
@@ -95,6 +129,12 @@ export const QueryIncidentsTool: ObservabilityTool = {
               state: incident.currentIncidentState?.name,
               severity: incident.incidentSeverity?.name,
               createdAt: incident.createdAt,
+              affectedMonitors: affectedMonitors || undefined,
+              labels: incidentLabels || undefined,
+              rootCause: incident.rootCause,
+              remediationNotes: incident.remediationNotes,
+              postmortemNote: incident.postmortemNote,
+              telemetryWindowStart: incident.createdAt,
             },
           ]
         : [];
@@ -202,3 +242,145 @@ export const QueryIncidentsTool: ObservabilityTool = {
     };
   },
 };
+
+export const SearchIncidentsTool: ObservabilityTool = {
+  name: "search_incidents",
+  description:
+    "Search PAST incidents by free text across their title, description, root cause, remediation notes and postmortem. Use this to ground root-cause analysis in history — e.g. 'have we seen this error before, and how was it resolved?'. Returns matching incidents (most recent first) with their root cause and postmortem so you can reuse prior resolutions.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      searchText: {
+        type: "string",
+        description:
+          "Text to match (case-insensitive substring) across incident title, description, root cause, remediation notes and postmortem.",
+      },
+      createdWithinHours: {
+        type: "number",
+        description:
+          "Only incidents created within this many hours (default 2160 = 90 days, max 8760 = 1 year).",
+      },
+      limit: {
+        type: "number",
+        description: "Maximum incidents to return (default 5, max 15).",
+      },
+    },
+    required: ["searchText"],
+  },
+  get requiredPermissions(): Array<Permission> {
+    return resolveReadPermissions();
+  },
+  execute: async (
+    args: JSONObject,
+    ctx: ToolContext,
+  ): Promise<ToolExecutionResult> => {
+    const searchText: string | undefined = ToolArgs.getString(
+      args,
+      "searchText",
+    );
+
+    if (!searchText) {
+      return {
+        dataForLlm:
+          "Error: searchText is required. Provide the text to search past incidents for.",
+        rowCount: 0,
+        citationLabel: "Incident search (no query)",
+        redactionCount: 0,
+        isTruncated: false,
+      };
+    }
+
+    const createdWithinHours: number = ToolArgs.getNumber(
+      args,
+      "createdWithinHours",
+      { defaultValue: 2160, min: 1, max: 8760 },
+    );
+    const limit: number = ToolArgs.getNumber(args, "limit", {
+      defaultValue: 5,
+      min: 1,
+      max: 15,
+    });
+
+    const endTime: Date = OneUptimeDate.getCurrentDate();
+    const startTime: Date = OneUptimeDate.addRemoveHours(
+      endTime,
+      -1 * createdWithinHours,
+    );
+
+    const incidents: Array<Incident> = await IncidentService.findBy({
+      query: {
+        createdAt: QueryHelper.inBetween(startTime, endTime),
+        /*
+         * The OR-joined ILIKE Raw references the listed columns, not the
+         * attached property — this is the established multiSearch attachment
+         * pattern (see QueryUtil).
+         */
+        title: QueryHelper.multiSearch(
+          [
+            "title",
+            "description",
+            "rootCause",
+            "remediationNotes",
+            "postmortemNote",
+          ],
+          searchText,
+        ),
+      },
+      select: {
+        _id: true,
+        title: true,
+        incidentNumber: true,
+        createdAt: true,
+        currentIncidentState: {
+          name: true,
+        },
+        incidentSeverity: {
+          name: true,
+        },
+        rootCause: true,
+        postmortemNote: true,
+      },
+      sort: {
+        createdAt: SortOrder.Descending,
+      },
+      limit: limit,
+      skip: 0,
+      props: ctx.props,
+    });
+
+    const rows: Array<JSONObject> = incidents.map((incident: Incident) => {
+      return {
+        id: incident.id?.toString(),
+        incidentNumber: incident.incidentNumber,
+        title: incident.title,
+        state: incident.currentIncidentState?.name,
+        severity: incident.incidentSeverity?.name,
+        createdAt: incident.createdAt,
+        rootCause: incident.rootCause,
+        postmortemNote: incident.postmortemNote,
+      };
+    });
+
+    const serialized: SerializedResult =
+      ToolResultSerializer.serializeRows(rows);
+
+    return {
+      dataForLlm: serialized.text,
+      rowCount: serialized.rowCount,
+      citationLabel: `Incident search "${searchText}" (${serialized.rowCount} found)`,
+      citationTarget: {
+        type: AIChatCitationTargetType.Incidents,
+      },
+      redactionCount: serialized.redactionCount,
+      isTruncated: serialized.isTruncated,
+      widget:
+        rows.length > 0
+          ? WidgetBuilder.incidentList({
+              title: `Incidents matching "${searchText}" (${rows.length})`,
+              items: rows,
+              link: { type: AIChatCitationTargetType.Incidents },
+            })
+          : undefined,
+    };
+  },
+};

package/Server/Utils/AI/Toolbox/Index.ts

@@ -13,11 +13,12 @@ import {
   ToolContext,
   ToolExecutionResult,
 } from "./ToolTypes";
-import { QueryIncidentsTool } from "./IncidentTools";
+import { QueryIncidentsTool, SearchIncidentsTool } from "./IncidentTools";
 import { QueryAlertsTool } from "./AlertTools";
 import { QueryMonitorsTool } from "./MonitorTools";
 import { TopExceptionsTool } from "./ExceptionTools";
 import { LogHistogramTool, SearchLogsTool } from "./LogTools";
+import { RecentChangesTool } from "./RecentChangesTools";
 import { QueryMetricsTool } from "./MetricTools";
 import { GetTraceTool, QueryTracesTool } from "./TraceTools";
 import { LookupContextTool } from "./ContextTools";
@@ -27,6 +28,12 @@ import {
   ResolveIncidentTool,
 } from "./IncidentWriteTools";
 import { AcknowledgeAlertTool, ResolveAlertTool } from "./AlertWriteTools";
+import {
+  PageOnCallPolicyTool,
+  RunRunbookTool,
+  PostIncidentStatusUpdateTool,
+  ChangeIncidentSeverityTool,
+} from "./SentinelActionTools";
 import AIChatPermissionMode from "../../../../Types/AI/AIChatPermissionMode";
 
 export interface ToolCallOutcome {
@@ -53,6 +60,7 @@ export default class AIToolbox {
   private static readonly tools: Array<ObservabilityTool> = [
     LookupContextTool,
     QueryIncidentsTool,
+    SearchIncidentsTool,
     QueryAlertsTool,
     QueryMonitorsTool,
     TopExceptionsTool,
@@ -61,12 +69,18 @@ export default class AIToolbox {
     QueryMetricsTool,
     QueryTracesTool,
     GetTraceTool,
+    RecentChangesTool,
     // Write tools (mutations). Gated by conversation permission mode.
     CreateIncidentTool,
     AcknowledgeIncidentTool,
     ResolveIncidentTool,
     AcknowledgeAlertTool,
     ResolveAlertTool,
+    // Sentinel action belt (Phase 0) — operate the platform, not just answer.
+    PageOnCallPolicyTool,
+    RunRunbookTool,
+    PostIncidentStatusUpdateTool,
+    ChangeIncidentSeverityTool,
   ];
 
   public static getTools(): Array<ObservabilityTool> {

package/Server/Utils/AI/Toolbox/RecentChangesTools.ts

@@ -0,0 +1,208 @@
+import Monitor from "../../../../Models/DatabaseModels/Monitor";
+import MonitorStatusTimeline from "../../../../Models/DatabaseModels/MonitorStatusTimeline";
+import ScheduledMaintenance from "../../../../Models/DatabaseModels/ScheduledMaintenance";
+import TelemetryException from "../../../../Models/DatabaseModels/TelemetryException";
+import { JSONObject } from "../../../../Types/JSON";
+import Permission from "../../../../Types/Permission";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import MonitorStatusTimelineService from "../../../Services/MonitorStatusTimelineService";
+import ScheduledMaintenanceService from "../../../Services/ScheduledMaintenanceService";
+import TelemetryExceptionService from "../../../Services/TelemetryExceptionService";
+import QueryHelper from "../../../Types/Database/QueryHelper";
+import logger from "../../Logger";
+import ToolResultSerializer, { SerializedResult } from "./Serializer";
+import {
+  ObservabilityTool,
+  ToolArgs,
+  ToolContext,
+  ToolExecutionResult,
+  TimeRangeSchemaProperties,
+} from "./ToolTypes";
+
+/*
+ * Gate on Monitor read permissions. This is defense-in-depth only: every
+ * source query below runs under the requesting user's props, so a user who
+ * cannot see monitors / exceptions / maintenance simply gets no rows from that
+ * source — never another user's data. Resolved lazily to avoid the
+ * import-time circular-dependency TypeError other tools document.
+ */
+let cachedReadPermissions: Array<Permission> | null = null;
+const resolveReadPermissions: () => Array<Permission> =
+  (): Array<Permission> => {
+    if (!cachedReadPermissions) {
+      cachedReadPermissions = new Monitor().getReadPermissions();
+    }
+    return cachedReadPermissions;
+  };
+
+// One normalized entry in the merged "what changed" feed.
+interface ChangeEvent {
+  at: Date;
+  change: string;
+  detail: string;
+}
+
+export const RecentChangesTool: ObservabilityTool = {
+  name: "recent_changes",
+  description:
+    "Get a single chronological feed of what changed in a time window: newly-seen exceptions, monitor status changes (up/down), and scheduled maintenance overlapping the window. This is the highest-yield way to answer 'what changed right before this started?' during root-cause analysis — call it with the window around when a problem began.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      ...TimeRangeSchemaProperties,
+      limitPerSource: {
+        type: "number",
+        description:
+          "Maximum items to pull from each source (exceptions, monitor changes, maintenance) before merging. Default 15, max 30.",
+      },
+    },
+  },
+  get requiredPermissions(): Array<Permission> {
+    return resolveReadPermissions();
+  },
+  execute: async (
+    args: JSONObject,
+    ctx: ToolContext,
+  ): Promise<ToolExecutionResult> => {
+    const timeRange: { startTime: Date; endTime: Date } = ToolArgs.getTimeRange(
+      args,
+      { defaultHours: 24, maxDays: 30 },
+    );
+    const startTime: Date = timeRange.startTime;
+    const endTime: Date = timeRange.endTime;
+
+    const limitPerSource: number = ToolArgs.getNumber(args, "limitPerSource", {
+      defaultValue: 15,
+      min: 1,
+      max: 30,
+    });
+
+    const events: Array<ChangeEvent> = [];
+
+    // 1) Newly-seen exceptions — their first occurrence falls inside the window.
+    try {
+      const exceptions: Array<TelemetryException> =
+        await TelemetryExceptionService.findBy({
+          query: {
+            firstSeenAt: QueryHelper.inBetween(startTime, endTime),
+          } as never,
+          select: {
+            message: true,
+            exceptionType: true,
+            firstSeenAt: true,
+            occuranceCount: true,
+          },
+          sort: { firstSeenAt: SortOrder.Descending },
+          limit: limitPerSource,
+          skip: 0,
+          props: ctx.props,
+        });
+
+      for (const exception of exceptions) {
+        if (!exception.firstSeenAt) {
+          continue;
+        }
+        events.push({
+          at: exception.firstSeenAt,
+          change: "new_exception",
+          detail: `${exception.exceptionType || "Exception"}: ${exception.message || ""} (x${exception.occuranceCount ?? 0})`,
+        });
+      }
+    } catch (error) {
+      logger.debug(`recent_changes: exceptions source skipped: ${error}`);
+    }
+
+    // 2) Monitor status changes (up/down) inside the window.
+    try {
+      const statusChanges: Array<MonitorStatusTimeline> =
+        await MonitorStatusTimelineService.findBy({
+          query: {
+            createdAt: QueryHelper.inBetween(startTime, endTime),
+          },
+          select: {
+            createdAt: true,
+            monitor: { name: true },
+            monitorStatus: { name: true },
+          },
+          sort: { createdAt: SortOrder.Descending },
+          limit: limitPerSource,
+          skip: 0,
+          props: ctx.props,
+        });
+
+      for (const statusChange of statusChanges) {
+        if (!statusChange.createdAt) {
+          continue;
+        }
+        events.push({
+          at: statusChange.createdAt,
+          change: "monitor_status_change",
+          detail: `${statusChange.monitor?.name || "Monitor"} → ${statusChange.monitorStatus?.name || "unknown"}`,
+        });
+      }
+    } catch (error) {
+      logger.debug(`recent_changes: monitor source skipped: ${error}`);
+    }
+
+    /*
+     * 3) Scheduled maintenance overlapping the window (started before it ended,
+     *    ends after it started).
+     */
+    try {
+      const maintenanceEvents: Array<ScheduledMaintenance> =
+        await ScheduledMaintenanceService.findBy({
+          query: {
+            startsAt: QueryHelper.lessThanEqualTo(endTime),
+            endsAt: QueryHelper.greaterThanEqualTo(startTime),
+          },
+          select: {
+            title: true,
+            startsAt: true,
+            endsAt: true,
+            currentScheduledMaintenanceState: { name: true },
+          },
+          sort: { startsAt: SortOrder.Descending },
+          limit: limitPerSource,
+          skip: 0,
+          props: ctx.props,
+        });
+
+      for (const maintenance of maintenanceEvents) {
+        if (!maintenance.startsAt) {
+          continue;
+        }
+        events.push({
+          at: maintenance.startsAt,
+          change: "scheduled_maintenance",
+          detail: `${maintenance.title || "Maintenance"} (${maintenance.currentScheduledMaintenanceState?.name || "scheduled"})`,
+        });
+      }
+    } catch (error) {
+      logger.debug(`recent_changes: maintenance source skipped: ${error}`);
+    }
+
+    // Merge into one chronological feed, most recent first.
+    events.sort((a: ChangeEvent, b: ChangeEvent) => {
+      return b.at.getTime() - a.at.getTime();
+    });
+
+    const rows: Array<JSONObject> = events.map((event: ChangeEvent) => {
+      return {
+        at: event.at,
+        change: event.change,
+        detail: event.detail,
+      };
+    });
+
+    const serialized: SerializedResult =
+      ToolResultSerializer.serializeRows(rows);
+
+    return {
+      dataForLlm: serialized.text,
+      rowCount: serialized.rowCount,
+      citationLabel: `Changes ${startTime.toISOString()} → ${endTime.toISOString()} (${serialized.rowCount} events)`,
+      redactionCount: serialized.redactionCount,
+      isTruncated: serialized.isTruncated,
+    };
+  },
+};