@oneuptime/common 11.5.4 → 11.5.5

package/Server/Utils/AI/{Sentinel → SRE}/IncidentInvestigationRunner.ts

@@ -5,28 +5,32 @@ import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/Inciden
 import IncidentInternalNote from "../../../../Models/DatabaseModels/IncidentInternalNote";
 import IncidentFeedService from "../../../Services/IncidentFeedService";
 import IncidentInternalNoteService from "../../../Services/IncidentInternalNoteService";
+import IncidentService from "../../../Services/IncidentService";
 import IncidentAIContextBuilder, {
   IncidentContextData,
 } from "../IncidentAIContextBuilder";
-import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
-import SentinelInvestigationQueue from "./InvestigationQueue";
-import SentinelMemory from "./SentinelMemory";
+import { AI_INCIDENT_INVESTIGATION_FEATURE } from "../../../Services/AIService";
+import AIInvestigationEngine from "./AIInvestigationEngine";
+import AIInvestigationQueue from "./InvestigationQueue";
+import AIConfidenceSignal, { ConfidenceSignal } from "./ConfidenceSignal";
+import AIMemory from "./AIMemory";
+import InstrumentationTaskTrigger from "./InstrumentationTaskTrigger";
 import { ObservabilityAssistantResult } from "../Chat/ObservabilityAssistant";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
 /*
- * Sentinel — incident investigation.
+ * AI SRE — incident investigation.
  *
- * When a new incident is declared, wakes Sentinel to investigate it (read-only)
+ * When a new incident is declared, wakes AI to investigate it (read-only)
  * and post a cited root-cause analysis into the incident timeline + Slack/Teams
  * — before the on-call engineer has finished reading the page. The trigger only
- * records durable intent (a Queued AIRun via SentinelInvestigationQueue), so a
+ * records durable intent (a Queued AIRun via AIInvestigationQueue), so a
  * pod restart can no longer orphan an investigation; the heavy lifting (run
  * lifecycle, the agent loop, confidence gating) lives in the shared
- * SentinelInvestigationEngine. See Internal/Roadmap/AISentinelExecution.md.
+ * AIInvestigationEngine. See Internal/Roadmap/AISentinelExecution.md.
  */
-export default class SentinelIncidentInvestigationRunner {
+export default class AIIncidentInvestigationRunner {
   /*
    * Entry point called (fire-and-forget) from IncidentService.onCreateSuccess.
    * Cheap: gates + a Queued AIRun row; execution happens via the queue.
@@ -41,7 +45,7 @@ export default class SentinelIncidentInvestigationRunner {
 
     try {
       if (
-        !(await SentinelInvestigationEngine.isEnabledForProject(
+        !(await AIInvestigationEngine.isEnabledForProject(
           projectId,
           "Incident",
         ))
@@ -49,20 +53,20 @@ export default class SentinelIncidentInvestigationRunner {
         return;
       }
 
-      await SentinelInvestigationQueue.enqueue({
+      await AIInvestigationQueue.enqueue({
         projectId,
         subjectIncidentId: incidentId,
       });
     } catch (error) {
       logger.error(
-        `Sentinel: unexpected error enqueueing investigation for incident ${incidentId.toString()}: ${error}`,
+        `AI: unexpected error enqueueing investigation for incident ${incidentId.toString()}: ${error}`,
       );
     }
   }
 
   /*
    * Execute a claimed run: assemble incident context and hand it to the
-   * engine. Called by SentinelInvestigationQueue after a successful claim.
+   * engine. Called by AIInvestigationQueue after a successful claim.
    * Never throws — failures are handed to the queue's retry policy so a
    * claimed run is always finalized or requeued.
    */
@@ -85,11 +89,11 @@ export default class SentinelIncidentInvestigationRunner {
 
       /*
        * Recurrence memory: pull in past resolved incidents that share this
-       * incident's monitors/labels so Sentinel can recognise "I've seen this
+       * incident's monitors/labels so AI can recognise "I've seen this
        * before" and reference the prior fix.
        */
       const priorCasesContext: string =
-        await SentinelMemory.getPriorSimilarIncidentsContext({
+        await AIMemory.getPriorSimilarIncidentsContext({
           projectId,
           currentIncidentId: incidentId,
           monitorNames: (contextData.incident.monitors || [])
@@ -111,7 +115,7 @@ export default class SentinelIncidentInvestigationRunner {
        * Context assembly failed — the run is claimed, so hand it to the
        * retry policy rather than leaving it Running until the sweeper.
        */
-      await SentinelInvestigationQueue.failOrRequeue({
+      await AIInvestigationQueue.failOrRequeue({
         aiRunId,
         attemptCount,
         errorMessage: `Failed to build incident context: ${
@@ -122,16 +126,17 @@ export default class SentinelIncidentInvestigationRunner {
       return;
     }
 
-    await SentinelInvestigationEngine.executeRun({
+    await AIInvestigationEngine.executeRun({
       aiRunId,
       projectId,
       attemptCount,
       request: {
-        feature: "Sentinel Incident Investigation",
+        // Persisted LlmLog label — owned by AIService (G4 budget list).
+        feature: AI_INCIDENT_INVESTIGATION_FEATURE,
         contextSummary,
         postAnalysis: async (postData: {
           analysisMarkdown: string;
-          isConfident: boolean;
+          confidence: ConfidenceSignal;
           result: ObservabilityAssistantResult;
         }): Promise<void> => {
           await IncidentFeedService.createIncidentFeedItem({
@@ -142,10 +147,15 @@ export default class SentinelIncidentInvestigationRunner {
             feedInfoInMarkdown: postData.analysisMarkdown,
             /*
              * Quiet mode: an inconclusive investigation posts to the timeline
-             * but does NOT loudly ping the workspace / on-call.
+             * but does NOT loudly ping the workspace / on-call. Fail
+             * direction (G6): when the confidence classification itself
+             * failed, the ping SENDS — quiet mode fails louder, not silent.
              */
             workspaceNotification: {
-              sendWorkspaceNotification: postData.isConfident,
+              sendWorkspaceNotification:
+                AIConfidenceSignal.shouldSendWorkspaceNotification(
+                  postData.confidence,
+                ),
             },
           });
 
@@ -173,7 +183,47 @@ export default class SentinelIncidentInvestigationRunner {
             });
           } catch (error) {
             logger.error(
-              `Sentinel: failed to post RCA internal note for incident ${incidentId.toString()}: ${error}`,
+              `AI: failed to post RCA internal note for incident ${incidentId.toString()}: ${error}`,
+            );
+          }
+
+          /*
+           * Measurement layer: record the time-to-rca metric (seconds from
+           * incident creation to this analysis post, with the same base
+           * attribute shape as every other incident metric). Best-effort —
+           * metrics must never break the RCA post.
+           */
+          try {
+            await IncidentService.recordTimeToRootCausePostedMetric({
+              incidentId,
+            });
+          } catch (error) {
+            logger.error(
+              `AI: failed to record time-to-rca metric for incident ${incidentId.toString()}: ${error}`,
+            );
+          }
+
+          /*
+           * Inconclusive means the telemetry was insufficient — for
+           * opted-in projects (Project.enableInstrumentationFixTasks,
+           * default false), queue an ImproveInstrumentation fix task that
+           * opens a PR adding the missing observability. Runs strictly
+           * AFTER the analysis is posted, and the trigger never throws, so
+           * the investigation can neither be blocked nor failed by it.
+           * Fail direction (G6): only a POSITIVE inconclusive verdict
+           * (deterministic floor or classification) enqueues — a failed
+           * classification does NOT create a PR.
+           */
+          if (
+            AIConfidenceSignal.shouldEnqueueInstrumentationTask(
+              postData.confidence,
+            )
+          ) {
+            await InstrumentationTaskTrigger.enqueueForInconclusiveInvestigation(
+              {
+                projectId,
+                incidentId,
+              },
             );
           }
         },

package/Server/Utils/AI/{Sentinel → SRE}/IncidentPostmortemRunner.ts

@@ -4,14 +4,14 @@ import Incident from "../../../../Models/DatabaseModels/Incident";
 import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
 import IncidentService from "../../../Services/IncidentService";
 import IncidentFeedService from "../../../Services/IncidentFeedService";
-import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
+import AIInvestigationEngine from "./AIInvestigationEngine";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
 /*
- * Sentinel — auto-draft postmortem on resolve (Phase 3, "close the loop").
+ * AI SRE — auto-draft postmortem on resolve (Phase 3, "close the loop").
  *
- * When an incident moves to a resolved state, Sentinel drafts a postmortem from
+ * When an incident moves to a resolved state, AI drafts a postmortem from
  * the incident timeline + telemetry (reusing the existing
  * IncidentService.generatePostmortemFromAI) and saves it to the incident for a
  * human to review and edit — turning a ~90-minute manual writeup into a review.
@@ -22,7 +22,7 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  */
 const MAX_FEED_PREVIEW_CHARS: number = 6000;
 
-export default class SentinelIncidentPostmortemRunner {
+export default class AIIncidentPostmortemRunner {
   @CaptureSpan()
   public static async draftPostmortemOnResolve(data: {
     incidentId: ObjectID;
@@ -32,7 +32,7 @@ export default class SentinelIncidentPostmortemRunner {
 
     try {
       if (
-        !(await SentinelInvestigationEngine.isEnabledForProject(
+        !(await AIInvestigationEngine.isEnabledForProject(
           projectId,
           "Incident",
         ))
@@ -83,7 +83,7 @@ export default class SentinelIncidentPostmortemRunner {
         projectId,
         incidentFeedEventType: IncidentFeedEventType.PostmortemNote,
         displayColor: Blue500,
-        feedInfoInMarkdown: `## 🧠 Sentinel — Draft Postmortem\n\nSentinel drafted a postmortem for incident #${incident.incidentNumber} from the incident timeline and telemetry. It has been saved on the incident for you to review and edit.`,
+        feedInfoInMarkdown: `## 🧠 AI — Draft Postmortem\n\nOneUptime AI drafted a postmortem for incident #${incident.incidentNumber} from the incident timeline and telemetry. It has been saved on the incident for you to review and edit.`,
         moreInformationInMarkdown:
           draft.length > MAX_FEED_PREVIEW_CHARS
             ? `${draft.substring(0, MAX_FEED_PREVIEW_CHARS)}\n\n…(truncated — the full draft is saved on the incident.)`
@@ -94,11 +94,11 @@ export default class SentinelIncidentPostmortemRunner {
       });
 
       logger.debug(
-        `Sentinel: drafted postmortem for incident ${incidentId.toString()}.`,
+        `AI: drafted postmortem for incident ${incidentId.toString()}.`,
       );
     } catch (error) {
       logger.error(
-        `Sentinel: failed to draft postmortem for incident ${incidentId.toString()}: ${error}`,
+        `AI: failed to draft postmortem for incident ${incidentId.toString()}: ${error}`,
       );
     }
   }

package/Server/Utils/AI/SRE/Insights/Detectors/ErrorLogSpikeDetector.ts

@@ -0,0 +1,364 @@
+import ObjectID from "../../../../../../Types/ObjectID";
+import OneUptimeDate from "../../../../../../Types/Date";
+import AIInsightType from "../../../../../../Types/AI/AIInsightType";
+import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
+import Service from "../../../../../../Models/DatabaseModels/Service";
+import LogAggregationService, {
+  AnalyticsTopItem,
+  HistogramBucket,
+} from "../../../../../Services/LogAggregationService";
+import ServiceService from "../../../../../Services/ServiceService";
+import {
+  InsightCandidate,
+  InsightDetector,
+  InsightScanContext,
+} from "../Types";
+
+/*
+ * ErrorLogSpike — the project's Error/Fatal log volume in the last hour
+ * spiked well above its own average hourly rate over the prior 24 hours.
+ *
+ * Two-stage on purpose (cost): stage 1 is ONE project-wide severity
+ * histogram, which ClickHouse serves from the proj_severity_histogram
+ * projection because severityText and the minute bucket are the
+ * projection's own keys. Only when stage 1 finds a spike does stage 2 run
+ * the service-attribution query — a top-list grouped by primaryEntityId,
+ * which falls back to a base-table scan but is bounded to the single spike
+ * hour. Deterministic — no LLM anywhere in this detector.
+ */
+
+// The spike window, compared against the prior baseline window.
+export const ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES: number = 60;
+
+/*
+ * Baseline: the 24 hours before the spike window, averaged per hour.
+ * Self-baselining means chronically chatty projects don't alert on their
+ * normal error volume.
+ */
+export const ERROR_LOG_SPIKE_BASELINE_WINDOW_HOURS: number = 24;
+
+/*
+ * Absolute floor: below 100 errors/hour a "3x spike" can be 20→100 raw
+ * lines — routine flakiness, not an event worth a human's attention.
+ */
+export const ERROR_LOG_SPIKE_MIN_RECENT_COUNT: number = 100;
+
+/*
+ * Volume multiplier that counts as a spike. The baseline hourly average is
+ * floored at 1 (see evaluateSpike) so a silent project's first errors
+ * still register without producing infinite multipliers.
+ */
+export const ERROR_LOG_SPIKE_MIN_MULTIPLIER: number = 3;
+
+// A >= 10x spike is a step change, not a fluctuation — escalate to High.
+export const ERROR_LOG_SPIKE_HIGH_SEVERITY_MULTIPLIER: number = 10;
+
+/*
+ * Histogram bucket size. The projection is minute-grained, so any bucket
+ * size is projection-served; 5 minutes keeps the recent/prior split error
+ * bounded to one bucket (< 5 min of data) while returning only ~300 rows
+ * for the whole 25h window.
+ */
+export const ERROR_LOG_SPIKE_HISTOGRAM_BUCKET_MINUTES: number = 5;
+
+/*
+ * At most this many per-service insights per spike. More than 3 services
+ * spiking together is one platform event, not N service events — the
+ * project-wide evidence block carries the full attribution list anyway.
+ */
+export const ERROR_LOG_SPIKE_MAX_ATTRIBUTED_SERVICES: number = 3;
+
+// Severity levels that count as "errors" for this detector (OTel names).
+export const ERROR_LOG_SEVERITIES: Array<string> = ["Error", "Fatal"];
+
+/*
+ * A ClickHouse DateTime rendered with no zone designator: "YYYY-MM-DD
+ * hh:mm:ss" (the shape LogAggregationService.getHistogram reads out of the
+ * `bucket` column), optionally "T"-separated and/or fractional. Anchored on
+ * purpose — any string carrying a zone (…Z, …+05:30) must NOT match, because
+ * parseBucketTime appends a Z to whatever matches and a double-designator
+ * string parses to an Invalid Date (which splitHistogram would then skip,
+ * zeroing the bucket).
+ */
+const ZONE_LESS_DATETIME_REGEX: RegExp =
+  /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(\.\d+)?$/;
+
+export interface ErrorLogSpikeDecision {
+  isSpike: boolean;
+  // recent count / max(baseline hourly average, 1).
+  multiplier: number;
+  severity: AIInsightSeverity;
+  recentCount: number;
+  // prior-window count / ERROR_LOG_SPIKE_BASELINE_WINDOW_HOURS (unfloored).
+  baselineHourlyAverage: number;
+}
+
+export default class ErrorLogSpikeDetector implements InsightDetector {
+  public insightType: AIInsightType = AIInsightType.ErrorLogSpike;
+
+  /*
+   * ClickHouse serves DateTime columns as a zone-less wall clock in the
+   * column's own zone — UTC in every OneUptime deployment. `new Date()` reads
+   * that shape as the PROCESS's LOCAL time, so on a worker whose TZ is not UTC
+   * every bucket silently slides by the UTC offset and the recent/prior
+   * boundary below lands in the wrong place: positive offsets push the spike's
+   * buckets into the baseline (suppressing every spike while inflating the
+   * average it is compared against), negative offsets drag hours of baseline
+   * into the "last 60 minutes" (fabricating spikes whose counts cannot even
+   * reconcile with stage 2, which binds real Date params and queries the true
+   * hour). Normalizing to an explicit UTC instant makes the split depend only
+   * on the data, never on the worker's clock configuration.
+   *
+   * Zone-carrying strings are parsed verbatim — see ZONE_LESS_DATETIME_REGEX.
+   * Anything else stays an Invalid Date so splitHistogram's isNaN guard still
+   * skips it. Same idiom as the dashboard's EdgeDetailPanel.parseBucketStart.
+   */
+  public static parseBucketTime(time: string): Date {
+    const trimmedTime: string = time.trim();
+
+    if (ZONE_LESS_DATETIME_REGEX.test(trimmedTime)) {
+      return new Date(`${trimmedTime.replace(" ", "T")}Z`);
+    }
+
+    return new Date(trimmedTime);
+  }
+
+  /*
+   * Pure split of one histogram into the recent window and the prior
+   * baseline window. A bucket belongs to "recent" when it STARTS at or
+   * after the boundary; the boundary-straddling bucket therefore counts as
+   * prior — a deliberate conservative bias (understates the spike by at
+   * most one bucket of data rather than inflating it).
+   */
+  public static splitHistogram(
+    buckets: Array<HistogramBucket>,
+    recentWindowStart: Date,
+  ): { recentCount: number; priorCount: number } {
+    let recentCount: number = 0;
+    let priorCount: number = 0;
+
+    for (const bucket of buckets) {
+      const bucketTime: Date = ErrorLogSpikeDetector.parseBucketTime(
+        bucket.time,
+      );
+      if (isNaN(bucketTime.getTime())) {
+        continue;
+      }
+      if (bucketTime.getTime() >= recentWindowStart.getTime()) {
+        recentCount += bucket.count;
+      } else {
+        priorCount += bucket.count;
+      }
+    }
+
+    return { recentCount, priorCount };
+  }
+
+  /*
+   * Pure decision: recent-hour error count vs the prior 24h window count.
+   * Spike ⇔ recent >= ERROR_LOG_SPIKE_MIN_RECENT_COUNT AND
+   * recent >= ERROR_LOG_SPIKE_MIN_MULTIPLIER * max(prior/24, 1).
+   */
+  public static evaluateSpike(
+    recentCount: number,
+    priorWindowCount: number,
+  ): ErrorLogSpikeDecision {
+    const baselineHourlyAverage: number =
+      priorWindowCount / ERROR_LOG_SPIKE_BASELINE_WINDOW_HOURS;
+    const effectiveBaseline: number = Math.max(baselineHourlyAverage, 1);
+    const multiplier: number = recentCount / effectiveBaseline;
+
+    const isSpike: boolean =
+      recentCount >= ERROR_LOG_SPIKE_MIN_RECENT_COUNT &&
+      multiplier >= ERROR_LOG_SPIKE_MIN_MULTIPLIER;
+
+    return {
+      isSpike,
+      multiplier,
+      severity:
+        multiplier >= ERROR_LOG_SPIKE_HIGH_SEVERITY_MULTIPLIER
+          ? AIInsightSeverity.High
+          : AIInsightSeverity.Medium,
+      recentCount,
+      baselineHourlyAverage,
+    };
+  }
+
+  /*
+   * Stable dedupe key: per attributed service, or "project" for the
+   * unattributed project-wide spike.
+   */
+  public static buildFingerprint(serviceId: string | undefined): string {
+    return `error-log-spike:${serviceId || "project"}`;
+  }
+
+  public async detect(
+    context: InsightScanContext,
+  ): Promise<Array<InsightCandidate>> {
+    const recentWindowStart: Date = OneUptimeDate.addRemoveMinutes(
+      context.now,
+      -1 * ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES,
+    );
+    const baselineWindowStart: Date = OneUptimeDate.addRemoveHours(
+      recentWindowStart,
+      -1 * ERROR_LOG_SPIKE_BASELINE_WINDOW_HOURS,
+    );
+
+    /*
+     * Stage 1 — projection-fast project-wide severity histogram over the
+     * whole 25h (baseline + spike) range. severityTexts stays on the
+     * projection; adding any service/entity filter here would silently
+     * force a base-table scan (LogAggregationService.appendCommonFilters).
+     */
+    const buckets: Array<HistogramBucket> =
+      await LogAggregationService.getHistogram({
+        projectId: context.projectId,
+        startTime: baselineWindowStart,
+        endTime: context.now,
+        bucketSizeInMinutes: ERROR_LOG_SPIKE_HISTOGRAM_BUCKET_MINUTES,
+        severityTexts: ERROR_LOG_SEVERITIES,
+      });
+
+    const { recentCount, priorCount } = ErrorLogSpikeDetector.splitHistogram(
+      buckets,
+      recentWindowStart,
+    );
+
+    const decision: ErrorLogSpikeDecision = ErrorLogSpikeDetector.evaluateSpike(
+      recentCount,
+      priorCount,
+    );
+
+    if (!decision.isSpike) {
+      return [];
+    }
+
+    /*
+     * Stage 2 — service attribution, only now that a spike exists. This
+     * top-list groups by primaryEntityId, which rejects the projection and
+     * scans the base table — acceptable because the scan is bounded to the
+     * single spike hour.
+     */
+    const topItems: Array<AnalyticsTopItem> =
+      await LogAggregationService.getAnalyticsTopList({
+        projectId: context.projectId,
+        startTime: recentWindowStart,
+        endTime: context.now,
+        bucketSizeInMinutes: ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES,
+        chartType: "toplist",
+        groupBy: ["primaryEntityId"],
+        aggregation: "count",
+        severityTexts: ERROR_LOG_SEVERITIES,
+        limit: ERROR_LOG_SPIKE_MAX_ATTRIBUTED_SERVICES,
+      });
+
+    /*
+     * Resolve entity ids to Service names. Non-Service entities (hosts,
+     * clusters) resolve to null and keep their raw id as the label — the
+     * evidence must stand alone even when attribution is imperfect.
+     */
+    const topServices: Array<{
+      serviceId: string;
+      serviceName: string;
+      isService: boolean;
+      count: number;
+    }> = [];
+
+    for (const item of topItems) {
+      const service: Service | null = await ServiceService.findOneById({
+        id: new ObjectID(item.value),
+        select: { name: true },
+        props: { isRoot: true },
+      });
+      topServices.push({
+        serviceId: item.value,
+        serviceName: service?.name || item.value,
+        isService: Boolean(service?.name),
+        count: item.count,
+      });
+    }
+
+    const detailLines: Array<string> = [
+      "**Error-log spike detected**",
+      "",
+      `- Error/Fatal logs in the last ${ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES} minutes: ${decision.recentCount}`,
+      `- Baseline: ${decision.baselineHourlyAverage.toFixed(2)}/hour over the prior ${ERROR_LOG_SPIKE_BASELINE_WINDOW_HOURS}h (${priorCount} total)`,
+      `- Spike multiplier: ${decision.multiplier.toFixed(1)}x`,
+    ];
+    if (topServices.length > 0) {
+      detailLines.push("", "Top contributing services in the spike window:");
+      for (const top of topServices) {
+        detailLines.push(`- ${top.serviceName}: ${top.count} error logs`);
+      }
+    }
+    const detailMarkdown: string = detailLines.join("\n");
+
+    const evidenceTopServices: Array<{ serviceName: string; count: number }> =
+      topServices.map(
+        (top: {
+          serviceId: string;
+          serviceName: string;
+          isService: boolean;
+          count: number;
+        }): { serviceName: string; count: number } => {
+          return { serviceName: top.serviceName, count: top.count };
+        },
+      );
+
+    /*
+     * One insight per top contributing service so each can be routed and
+     * triaged in its own context; when attribution found nothing, a single
+     * project-wide insight still surfaces the event.
+     */
+    if (topServices.length === 0) {
+      return [
+        {
+          insightType: AIInsightType.ErrorLogSpike,
+          fingerprint: ErrorLogSpikeDetector.buildFingerprint(undefined),
+          title: `Error-log spike: ${decision.multiplier.toFixed(1)}x normal volume across the project`,
+          detailMarkdown,
+          severity: decision.severity,
+          evidence: {
+            logSpike: {
+              recentErrorCount: decision.recentCount,
+              baselineHourlyAverage: decision.baselineHourlyAverage,
+              spikeMultiplier: decision.multiplier,
+              windowMinutes: ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES,
+              topServices: evidenceTopServices,
+            },
+          },
+        },
+      ];
+    }
+
+    return topServices.map(
+      (top: {
+        serviceId: string;
+        serviceName: string;
+        isService: boolean;
+        count: number;
+      }): InsightCandidate => {
+        return {
+          insightType: AIInsightType.ErrorLogSpike,
+          fingerprint: ErrorLogSpikeDetector.buildFingerprint(top.serviceId),
+          title: `Error-log spike: ${decision.multiplier.toFixed(1)}x normal volume in ${top.serviceName}`,
+          detailMarkdown,
+          severity: decision.severity,
+          serviceName: top.serviceName,
+          telemetryServiceId: top.isService
+            ? new ObjectID(top.serviceId)
+            : undefined,
+          evidence: {
+            logSpike: {
+              recentErrorCount: decision.recentCount,
+              baselineHourlyAverage: decision.baselineHourlyAverage,
+              spikeMultiplier: decision.multiplier,
+              windowMinutes: ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES,
+              topServices: evidenceTopServices,
+            },
+          },
+        };
+      },
+    );
+  }
+}