@oneuptime/common 11.3.26 → 11.3.28

package/Server/Services/UserOnCallLogTimelineService.ts

@@ -15,6 +15,8 @@ import UserNotificationExecutionStatus from "../../Types/UserNotification/UserNo
 import User from "../../Models/DatabaseModels/User";
 import Model from "../../Models/DatabaseModels/UserOnCallLogTimeline";
 import AlertService from "./AlertService";
+import AlertEpisodeService from "./AlertEpisodeService";
+import IncidentEpisodeService from "./IncidentEpisodeService";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -40,6 +42,8 @@ export class Service extends DatabaseService<Model> {
           onCallDutyPolicyExecutionLogId: true,
           triggeredByIncidentId: true,
           triggeredByAlertId: true,
+          triggeredByAlertEpisodeId: true,
+          triggeredByIncidentEpisodeId: true,
           onCallDutyPolicyExecutionLogTimelineId: true,
         },
         skip: 0,
@@ -52,6 +56,19 @@ export class Service extends DatabaseService<Model> {
       for (const item of items) {
         const isIncident: boolean = Boolean(item.triggeredByIncidentId);
         const isAlert: boolean = Boolean(item.triggeredByAlertId);
+        /*
+         * Alert-episode and incident-episode pages were never handled here, so
+         * acknowledging one via call/SMS/email/push marked only this user's log
+         * done but never acknowledged the EPISODE. ExecutePendingExecutions gates
+         * every co-notified user's continuation solely on isEpisodeAcknowledged,
+         * so the other responders kept escalating and the episode stayed
+         * unacknowledged in its own state (audit F15). Handle both episode kinds
+         * the same way incident/alert are handled below.
+         */
+        const isAlertEpisode: boolean = Boolean(item.triggeredByAlertEpisodeId);
+        const isIncidentEpisode: boolean = Boolean(
+          item.triggeredByIncidentEpisodeId,
+        );
 
         // this incident is acknowledged.
 
@@ -73,7 +90,16 @@ export class Service extends DatabaseService<Model> {
           throw new BadDataException("User not found.");
         }
 
-        const statusMessage: string = `${isIncident ? "Incident" : "Alert"} acknowledged by ${user.name} (${user.email})`;
+        const entityLabel: string =
+          isIncident || isIncidentEpisode
+            ? isIncidentEpisode
+              ? "Incident Episode"
+              : "Incident"
+            : isAlertEpisode
+              ? "Alert Episode"
+              : "Alert";
+
+        const statusMessage: string = `${entityLabel} acknowledged by ${user.name} (${user.email})`;
 
         await UserOnCallLogService.updateOneById({
           id: item.userNotificationLogId!,
@@ -133,6 +159,22 @@ export class Service extends DatabaseService<Model> {
             item.userId!,
           );
         }
+
+        if (isAlertEpisode) {
+          // alert episode — stops co-notified responders from escalating.
+          await AlertEpisodeService.acknowledgeEpisode(
+            item.triggeredByAlertEpisodeId!,
+            item.userId!,
+          );
+        }
+
+        if (isIncidentEpisode) {
+          // incident episode — stops co-notified responders from escalating.
+          await IncidentEpisodeService.acknowledgeEpisode(
+            item.triggeredByIncidentEpisodeId!,
+            item.userId!,
+          );
+        }
       }
     }
 

package/Server/Types/Database/QueryUtil.ts

@@ -549,6 +549,63 @@ export default class QueryUtil {
           _id: QueryHelper.any(query[key] as Array<string>),
         };
       }
+
+      /*
+       * Text-search operator nested under a single-Entity relation, e.g.
+       * { user: { name: new Search("foo") } }. serializeQuery otherwise only
+       * visits top-level keys, so the nested operator reaches TypeORM as a raw
+       * class instance and silently degrades to an exact equality
+       * (`user.name = 'foo'` instead of `user.name ILIKE '%foo%'`) — which is
+       * why the Owner / user picker "search" appeared to do nothing once a
+       * project had more than a page of members.
+       *
+       * We deliberately touch ONLY the ILIKE-family leaf operators here and
+       * leave every other nested value (arrays, ids, plain equality, other
+       * operators) exactly as it was, so existing nested relation filters (the
+       * access-control label filters built by @CanAccessIfCanReadOn, etc.)
+       * serialize unchanged.
+       */
+      if (
+        tableColumnMetadata &&
+        tableColumnMetadata.modelType &&
+        tableColumnMetadata.type === TableColumnType.Entity &&
+        query[key] &&
+        typeof query[key] === Typeof.Object &&
+        !Array.isArray(query[key]) &&
+        (Object.getPrototypeOf(query[key]) === Object.prototype ||
+          Object.getPrototypeOf(query[key]) === null)
+      ) {
+        const relationModel: BaseModel = new tableColumnMetadata.modelType();
+        const relationQuery: Record<string, any> = query[key] as Record<
+          string,
+          any
+        >;
+        for (const relationKey in relationQuery) {
+          const relationColumnMetadata: TableColumnMetadata =
+            relationModel.getTableColumnMetadata(relationKey);
+          if (!relationColumnMetadata) {
+            continue;
+          }
+          const nestedValue: any = relationQuery[relationKey];
+          if (nestedValue instanceof Search) {
+            relationQuery[relationKey] = QueryHelper.search(
+              (nestedValue as Search<string>).toString(),
+            ) as any;
+          } else if (nestedValue instanceof StartsWith) {
+            relationQuery[relationKey] = QueryHelper.startsWith(
+              (nestedValue as StartsWith<string>).toString(),
+            ) as any;
+          } else if (nestedValue instanceof EndsWith) {
+            relationQuery[relationKey] = QueryHelper.endsWith(
+              (nestedValue as EndsWith<string>).toString(),
+            ) as any;
+          } else if (nestedValue instanceof NotContains) {
+            relationQuery[relationKey] = QueryHelper.notContains(
+              (nestedValue as NotContains<string>).toString(),
+            ) as any;
+          }
+        }
+      }
     }
 
     return query;

package/Server/Utils/AI/AlertAIContextBuilder.ts

@@ -235,7 +235,12 @@ The note should:
 4. Be detailed enough to help team members understand the situation
 5. Use technical language appropriate for the engineering team
 
-Write in markdown format for better readability. Be thorough and technical.`;
+Write in markdown format for better readability. Be thorough and technical.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided alert data. Do NOT invent causes, metrics, impact, or actions the data does not support.
+- If something a section needs is not in the data, write "Not available in the alert record" instead of guessing.
+- Never fabricate numbers or state confidence levels.`;
     }
 
     // Build user message

package/Server/Utils/AI/Chat/ChatAgentRunner.ts

@@ -908,9 +908,21 @@ export default class ChatAgentRunner {
         continue;
       }
 
+      const isUserMessage: boolean = message.role === AIChatMessageRole.User;
+
       messages.push({
-        role: message.role === AIChatMessageRole.User ? "user" : "assistant",
-        content: message.contentInMarkdown,
+        role: isUserMessage ? "user" : "assistant",
+        /*
+         * A prior assistant answer carries [C#] citation markers that pointed
+         * at that turn's tool results — citations that no longer exist in this
+         * turn. Left in the replayed history the model echoes and renumbers
+         * them, and this turn's stripFabricatedCitationMarkers then deletes the
+         * unmatched ones, leaving claims that look uncited. Strip the markers
+         * from replayed answers so only freshly minted citations ever appear.
+         */
+        content: isUserMessage
+          ? message.contentInMarkdown
+          : message.contentInMarkdown.replace(/\s?\[C\d+\]/g, ""),
       });
     }
 

package/Server/Utils/AI/IncidentAIContextBuilder.ts

@@ -365,7 +365,12 @@ The postmortem should:
 
 Use a standard incident postmortem format with sections for: Executive Summary, Timeline, Root Cause Analysis, Impact, Action Items, and Lessons Learned.
 
-Write in a professional, clear, and concise manner. Use markdown formatting for better readability.`;
+Write in a professional, clear, and concise manner. Use markdown formatting for better readability.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided incident data. Do NOT invent root causes, timelines, metrics, customer impact, or actions the data does not support.
+- If a section needs information the data does not contain, write "Not available in the incident record" instead of guessing.
+- Never fabricate numbers or state confidence levels.`;
     }
 
     // Build user message based on whether template is provided
@@ -549,7 +554,12 @@ DO NOT include:
 - Confidential information
 - Excessive jargon
 
-Write in markdown format for better readability.`;
+Write in markdown format for better readability.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided incident data; never invent impact, causes, timing, or status.
+- If something is not known from the data, say it is still being investigated rather than guessing.
+- Never fabricate numbers or specific timings the data does not support.`;
       }
     } else if (template) {
       // Internal note with template
@@ -577,7 +587,12 @@ The note should:
 4. Be detailed enough to help team members understand the situation
 5. Use technical language appropriate for the engineering team
 
-Write in markdown format for better readability. Be thorough and technical.`;
+Write in markdown format for better readability. Be thorough and technical.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided incident data. Do NOT invent root causes, metrics, impact, or actions the data does not support.
+- If something a section needs is not in the data, write "Not available in the incident record" instead of guessing.
+- Never fabricate numbers or state confidence levels.`;
     }
 
     // Build user message

package/Server/Utils/AI/IncidentEpisodeAIContextBuilder.ts

@@ -389,7 +389,12 @@ The postmortem should:
 
 Use a standard incident postmortem format with sections for: Executive Summary, Timeline, Root Cause Analysis, Impact, Action Items, and Lessons Learned.
 
-Write in a professional, clear, and concise manner. Use markdown formatting for better readability.`;
+Write in a professional, clear, and concise manner. Use markdown formatting for better readability.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided episode data. Do NOT invent root causes, timelines, metrics, impact, or actions the data does not support.
+- If a section needs information the data does not contain, write "Not available in the incident record" instead of guessing.
+- Never fabricate numbers or state confidence levels.`;
     }
 
     // Build user message based on whether template is provided

package/Server/Utils/AI/ScheduledMaintenanceAIContextBuilder.ts

@@ -288,7 +288,12 @@ DO NOT include:
 - Confidential information
 - Excessive jargon
 
-Write in markdown format for better readability.`;
+Write in markdown format for better readability.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided maintenance data; never invent scope, impact, timing, or status.
+- If something is not known from the data, say it will be confirmed rather than guessing.
+- Never fabricate numbers or specific timings the data does not support.`;
       }
     } else if (template) {
       // Internal note with template
@@ -316,7 +321,12 @@ The note should:
 4. Be detailed enough to help team members understand the current status
 5. Use technical language appropriate for the engineering team
 
-Write in markdown format for better readability. Be thorough and technical.`;
+Write in markdown format for better readability. Be thorough and technical.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided maintenance data. Do NOT invent scope, metrics, impact, or actions the data does not support.
+- If something a section needs is not in the data, write "Not available in the maintenance record" instead of guessing.
+- Never fabricate numbers or state confidence levels.`;
     }
 
     // Build user message

package/Server/Utils/AI/Toolbox/IncidentTools.ts

@@ -35,7 +35,7 @@ const resolveReadPermissions: () => Array<Permission> =
 export const QueryIncidentsTool: ObservabilityTool = {
   name: "query_incidents",
   description:
-    "Query incidents in this project. Returns the most recent incidents with their current state and severity. Pass incidentId to get full details of one incident.",
+    "Query incidents in this project. Returns the most recent incidents with their current state and severity. Pass incidentId to get full details of one incident — including its affected monitors, labels, root cause, remediation and postmortem notes, and the incident window (telemetryWindowStart) to pivot into logs/traces/metrics for the affected services.",
   inputSchema: {
     type: "object",
     properties: {
@@ -81,10 +81,44 @@ export const QueryIncidentsTool: ObservabilityTool = {
           incidentSeverity: {
             name: true,
           },
+          monitors: {
+            _id: true,
+            name: true,
+          },
+          labels: {
+            name: true,
+          },
+          rootCause: true,
+          remediationNotes: true,
+          postmortemNote: true,
         },
         props: ctx.props,
       });
 
+      /*
+       * Surface the incident's blast radius (affected monitors + labels) and
+       * its own window so the model can pivot from an incident into the
+       * logs/traces/metrics of the affected services over the right time
+       * range, instead of guessing which service or hour to look at.
+       */
+      const affectedMonitors: string = (incident?.monitors || [])
+        .map((monitor: { name?: string }) => {
+          return monitor.name || "";
+        })
+        .filter((value: string) => {
+          return value.length > 0;
+        })
+        .join(", ");
+
+      const incidentLabels: string = (incident?.labels || [])
+        .map((label: { name?: string }) => {
+          return label.name || "";
+        })
+        .filter((value: string) => {
+          return value.length > 0;
+        })
+        .join(", ");
+
       const rows: Array<JSONObject> = incident
         ? [
             {
@@ -95,6 +129,12 @@ export const QueryIncidentsTool: ObservabilityTool = {
               state: incident.currentIncidentState?.name,
               severity: incident.incidentSeverity?.name,
               createdAt: incident.createdAt,
+              affectedMonitors: affectedMonitors || undefined,
+              labels: incidentLabels || undefined,
+              rootCause: incident.rootCause,
+              remediationNotes: incident.remediationNotes,
+              postmortemNote: incident.postmortemNote,
+              telemetryWindowStart: incident.createdAt,
             },
           ]
         : [];
@@ -202,3 +242,145 @@ export const QueryIncidentsTool: ObservabilityTool = {
     };
   },
 };
+
+export const SearchIncidentsTool: ObservabilityTool = {
+  name: "search_incidents",
+  description:
+    "Search PAST incidents by free text across their title, description, root cause, remediation notes and postmortem. Use this to ground root-cause analysis in history — e.g. 'have we seen this error before, and how was it resolved?'. Returns matching incidents (most recent first) with their root cause and postmortem so you can reuse prior resolutions.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      searchText: {
+        type: "string",
+        description:
+          "Text to match (case-insensitive substring) across incident title, description, root cause, remediation notes and postmortem.",
+      },
+      createdWithinHours: {
+        type: "number",
+        description:
+          "Only incidents created within this many hours (default 2160 = 90 days, max 8760 = 1 year).",
+      },
+      limit: {
+        type: "number",
+        description: "Maximum incidents to return (default 5, max 15).",
+      },
+    },
+    required: ["searchText"],
+  },
+  get requiredPermissions(): Array<Permission> {
+    return resolveReadPermissions();
+  },
+  execute: async (
+    args: JSONObject,
+    ctx: ToolContext,
+  ): Promise<ToolExecutionResult> => {
+    const searchText: string | undefined = ToolArgs.getString(
+      args,
+      "searchText",
+    );
+
+    if (!searchText) {
+      return {
+        dataForLlm:
+          "Error: searchText is required. Provide the text to search past incidents for.",
+        rowCount: 0,
+        citationLabel: "Incident search (no query)",
+        redactionCount: 0,
+        isTruncated: false,
+      };
+    }
+
+    const createdWithinHours: number = ToolArgs.getNumber(
+      args,
+      "createdWithinHours",
+      { defaultValue: 2160, min: 1, max: 8760 },
+    );
+    const limit: number = ToolArgs.getNumber(args, "limit", {
+      defaultValue: 5,
+      min: 1,
+      max: 15,
+    });
+
+    const endTime: Date = OneUptimeDate.getCurrentDate();
+    const startTime: Date = OneUptimeDate.addRemoveHours(
+      endTime,
+      -1 * createdWithinHours,
+    );
+
+    const incidents: Array<Incident> = await IncidentService.findBy({
+      query: {
+        createdAt: QueryHelper.inBetween(startTime, endTime),
+        /*
+         * The OR-joined ILIKE Raw references the listed columns, not the
+         * attached property — this is the established multiSearch attachment
+         * pattern (see QueryUtil).
+         */
+        title: QueryHelper.multiSearch(
+          [
+            "title",
+            "description",
+            "rootCause",
+            "remediationNotes",
+            "postmortemNote",
+          ],
+          searchText,
+        ),
+      },
+      select: {
+        _id: true,
+        title: true,
+        incidentNumber: true,
+        createdAt: true,
+        currentIncidentState: {
+          name: true,
+        },
+        incidentSeverity: {
+          name: true,
+        },
+        rootCause: true,
+        postmortemNote: true,
+      },
+      sort: {
+        createdAt: SortOrder.Descending,
+      },
+      limit: limit,
+      skip: 0,
+      props: ctx.props,
+    });
+
+    const rows: Array<JSONObject> = incidents.map((incident: Incident) => {
+      return {
+        id: incident.id?.toString(),
+        incidentNumber: incident.incidentNumber,
+        title: incident.title,
+        state: incident.currentIncidentState?.name,
+        severity: incident.incidentSeverity?.name,
+        createdAt: incident.createdAt,
+        rootCause: incident.rootCause,
+        postmortemNote: incident.postmortemNote,
+      };
+    });
+
+    const serialized: SerializedResult =
+      ToolResultSerializer.serializeRows(rows);
+
+    return {
+      dataForLlm: serialized.text,
+      rowCount: serialized.rowCount,
+      citationLabel: `Incident search "${searchText}" (${serialized.rowCount} found)`,
+      citationTarget: {
+        type: AIChatCitationTargetType.Incidents,
+      },
+      redactionCount: serialized.redactionCount,
+      isTruncated: serialized.isTruncated,
+      widget:
+        rows.length > 0
+          ? WidgetBuilder.incidentList({
+              title: `Incidents matching "${searchText}" (${rows.length})`,
+              items: rows,
+              link: { type: AIChatCitationTargetType.Incidents },
+            })
+          : undefined,
+    };
+  },
+};

package/Server/Utils/AI/Toolbox/Index.ts

@@ -13,11 +13,12 @@ import {
   ToolContext,
   ToolExecutionResult,
 } from "./ToolTypes";
-import { QueryIncidentsTool } from "./IncidentTools";
+import { QueryIncidentsTool, SearchIncidentsTool } from "./IncidentTools";
 import { QueryAlertsTool } from "./AlertTools";
 import { QueryMonitorsTool } from "./MonitorTools";
 import { TopExceptionsTool } from "./ExceptionTools";
 import { LogHistogramTool, SearchLogsTool } from "./LogTools";
+import { RecentChangesTool } from "./RecentChangesTools";
 import { QueryMetricsTool } from "./MetricTools";
 import { GetTraceTool, QueryTracesTool } from "./TraceTools";
 import { LookupContextTool } from "./ContextTools";
@@ -53,6 +54,7 @@ export default class AIToolbox {
   private static readonly tools: Array<ObservabilityTool> = [
     LookupContextTool,
     QueryIncidentsTool,
+    SearchIncidentsTool,
     QueryAlertsTool,
     QueryMonitorsTool,
     TopExceptionsTool,
@@ -61,6 +63,7 @@ export default class AIToolbox {
     QueryMetricsTool,
     QueryTracesTool,
     GetTraceTool,
+    RecentChangesTool,
     // Write tools (mutations). Gated by conversation permission mode.
     CreateIncidentTool,
     AcknowledgeIncidentTool,