@oneuptime/common 11.5.4 → 11.5.5

package/build/dist/Server/Utils/AI/{Sentinel → SRE}/IncidentInvestigationRunner.js

@@ -13,24 +13,28 @@ import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/Inciden
 import IncidentInternalNote from "../../../../Models/DatabaseModels/IncidentInternalNote";
 import IncidentFeedService from "../../../Services/IncidentFeedService";
 import IncidentInternalNoteService from "../../../Services/IncidentInternalNoteService";
+import IncidentService from "../../../Services/IncidentService";
 import IncidentAIContextBuilder from "../IncidentAIContextBuilder";
-import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
-import SentinelInvestigationQueue from "./InvestigationQueue";
-import SentinelMemory from "./SentinelMemory";
+import { AI_INCIDENT_INVESTIGATION_FEATURE } from "../../../Services/AIService";
+import AIInvestigationEngine from "./AIInvestigationEngine";
+import AIInvestigationQueue from "./InvestigationQueue";
+import AIConfidenceSignal from "./ConfidenceSignal";
+import AIMemory from "./AIMemory";
+import InstrumentationTaskTrigger from "./InstrumentationTaskTrigger";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 /*
- * Sentinel — incident investigation.
+ * AI SRE — incident investigation.
  *
- * When a new incident is declared, wakes Sentinel to investigate it (read-only)
+ * When a new incident is declared, wakes AI to investigate it (read-only)
  * and post a cited root-cause analysis into the incident timeline + Slack/Teams
  * — before the on-call engineer has finished reading the page. The trigger only
- * records durable intent (a Queued AIRun via SentinelInvestigationQueue), so a
+ * records durable intent (a Queued AIRun via AIInvestigationQueue), so a
  * pod restart can no longer orphan an investigation; the heavy lifting (run
  * lifecycle, the agent loop, confidence gating) lives in the shared
- * SentinelInvestigationEngine. See Internal/Roadmap/AISentinelExecution.md.
+ * AIInvestigationEngine. See Internal/Roadmap/AISentinelExecution.md.
  */
-export default class SentinelIncidentInvestigationRunner {
+export default class AIIncidentInvestigationRunner {
     /*
      * Entry point called (fire-and-forget) from IncidentService.onCreateSuccess.
      * Cheap: gates + a Queued AIRun row; execution happens via the queue.
@@ -39,21 +43,21 @@ export default class SentinelIncidentInvestigationRunner {
     static async investigateNewIncident(data) {
         const { incidentId, projectId } = data;
         try {
-            if (!(await SentinelInvestigationEngine.isEnabledForProject(projectId, "Incident"))) {
+            if (!(await AIInvestigationEngine.isEnabledForProject(projectId, "Incident"))) {
                 return;
             }
-            await SentinelInvestigationQueue.enqueue({
+            await AIInvestigationQueue.enqueue({
                 projectId,
                 subjectIncidentId: incidentId,
             });
         }
         catch (error) {
-            logger.error(`Sentinel: unexpected error enqueueing investigation for incident ${incidentId.toString()}: ${error}`);
+            logger.error(`AI: unexpected error enqueueing investigation for incident ${incidentId.toString()}: ${error}`);
         }
     }
     /*
      * Execute a claimed run: assemble incident context and hand it to the
-     * engine. Called by SentinelInvestigationQueue after a successful claim.
+     * engine. Called by AIInvestigationQueue after a successful claim.
      * Never throws — failures are handed to the queue's retry policy so a
      * claimed run is always finalized or requeued.
      */
@@ -67,10 +71,10 @@ export default class SentinelIncidentInvestigationRunner {
             });
             /*
              * Recurrence memory: pull in past resolved incidents that share this
-             * incident's monitors/labels so Sentinel can recognise "I've seen this
+             * incident's monitors/labels so AI can recognise "I've seen this
              * before" and reference the prior fix.
              */
-            const priorCasesContext = await SentinelMemory.getPriorSimilarIncidentsContext({
+            const priorCasesContext = await AIMemory.getPriorSimilarIncidentsContext({
                 projectId,
                 currentIncidentId: incidentId,
                 monitorNames: (contextData.incident.monitors || [])
@@ -92,7 +96,7 @@ export default class SentinelIncidentInvestigationRunner {
              * Context assembly failed — the run is claimed, so hand it to the
              * retry policy rather than leaving it Running until the sweeper.
              */
-            await SentinelInvestigationQueue.failOrRequeue({
+            await AIInvestigationQueue.failOrRequeue({
                 aiRunId,
                 attemptCount,
                 errorMessage: `Failed to build incident context: ${error instanceof Error ? error.message : String(error)}`,
@@ -100,12 +104,13 @@ export default class SentinelIncidentInvestigationRunner {
             });
             return;
         }
-        await SentinelInvestigationEngine.executeRun({
+        await AIInvestigationEngine.executeRun({
             aiRunId,
             projectId,
             attemptCount,
             request: {
-                feature: "Sentinel Incident Investigation",
+                // Persisted LlmLog label — owned by AIService (G4 budget list).
+                feature: AI_INCIDENT_INVESTIGATION_FEATURE,
                 contextSummary,
                 postAnalysis: async (postData) => {
                     await IncidentFeedService.createIncidentFeedItem({
@@ -116,10 +121,12 @@ export default class SentinelIncidentInvestigationRunner {
                         feedInfoInMarkdown: postData.analysisMarkdown,
                         /*
                          * Quiet mode: an inconclusive investigation posts to the timeline
-                         * but does NOT loudly ping the workspace / on-call.
+                         * but does NOT loudly ping the workspace / on-call. Fail
+                         * direction (G6): when the confidence classification itself
+                         * failed, the ping SENDS — quiet mode fails louder, not silent.
                          */
                         workspaceNotification: {
-                            sendWorkspaceNotification: postData.isConfident,
+                            sendWorkspaceNotification: AIConfidenceSignal.shouldSendWorkspaceNotification(postData.confidence),
                         },
                     });
                     /*
@@ -145,7 +152,38 @@ export default class SentinelIncidentInvestigationRunner {
                         });
                     }
                     catch (error) {
-                        logger.error(`Sentinel: failed to post RCA internal note for incident ${incidentId.toString()}: ${error}`);
+                        logger.error(`AI: failed to post RCA internal note for incident ${incidentId.toString()}: ${error}`);
+                    }
+                    /*
+                     * Measurement layer: record the time-to-rca metric (seconds from
+                     * incident creation to this analysis post, with the same base
+                     * attribute shape as every other incident metric). Best-effort —
+                     * metrics must never break the RCA post.
+                     */
+                    try {
+                        await IncidentService.recordTimeToRootCausePostedMetric({
+                            incidentId,
+                        });
+                    }
+                    catch (error) {
+                        logger.error(`AI: failed to record time-to-rca metric for incident ${incidentId.toString()}: ${error}`);
+                    }
+                    /*
+                     * Inconclusive means the telemetry was insufficient — for
+                     * opted-in projects (Project.enableInstrumentationFixTasks,
+                     * default false), queue an ImproveInstrumentation fix task that
+                     * opens a PR adding the missing observability. Runs strictly
+                     * AFTER the analysis is posted, and the trigger never throws, so
+                     * the investigation can neither be blocked nor failed by it.
+                     * Fail direction (G6): only a POSITIVE inconclusive verdict
+                     * (deterministic floor or classification) enqueues — a failed
+                     * classification does NOT create a PR.
+                     */
+                    if (AIConfidenceSignal.shouldEnqueueInstrumentationTask(postData.confidence)) {
+                        await InstrumentationTaskTrigger.enqueueForInconclusiveInvestigation({
+                            projectId,
+                            incidentId,
+                        });
                     }
                 },
             },
@@ -204,11 +242,11 @@ __decorate([
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
-], SentinelIncidentInvestigationRunner, "investigateNewIncident", null);
+], AIIncidentInvestigationRunner, "investigateNewIncident", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
-], SentinelIncidentInvestigationRunner, "executeInvestigation", null);
+], AIIncidentInvestigationRunner, "executeInvestigation", null);
 //# sourceMappingURL=IncidentInvestigationRunner.js.map

package/build/dist/Server/Utils/AI/SRE/IncidentInvestigationRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IncidentInvestigationRunner.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/SRE/IncidentInvestigationRunner.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,aAAa,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,+BAA+B,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gDAAgD,CAAC;AACvF,OAAO,oBAAoB,MAAM,wDAAwD,CAAC;AAC1F,OAAO,mBAAmB,MAAM,uCAAuC,CAAC;AACxE,OAAO,2BAA2B,MAAM,+CAA+C,CAAC;AACxF,OAAO,eAAe,MAAM,mCAAmC,CAAC;AAChE,OAAO,wBAEN,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,iCAAiC,EAAE,MAAM,6BAA6B,CAAC;AAChF,OAAO,qBAAqB,MAAM,yBAAyB,CAAC;AAC5D,OAAO,oBAAoB,MAAM,sBAAsB,CAAC;AACxD,OAAO,kBAAwC,MAAM,oBAAoB,CAAC;AAC1E,OAAO,QAAQ,MAAM,YAAY,CAAC;AAClC,OAAO,0BAA0B,MAAM,8BAA8B,CAAC;AAEtE,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAEtD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,OAAO,OAAO,6BAA6B;IAChD;;;;OAIG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAG1C;QACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEvC,IAAI,CAAC;YACH,IACE,CAAC,CAAC,MAAM,qBAAqB,CAAC,mBAAmB,CAC/C,SAAS,EACT,UAAU,CACX,CAAC,EACF,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,oBAAoB,CAAC,OAAO,CAAC;gBACjC,SAAS;gBACT,iBAAiB,EAAE,UAAU;aAC9B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,8DAA8D,UAAU,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CAChG,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAKxC;QACC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;QAE9D,IAAI,cAAsB,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,WAAW,GACf,MAAM,wBAAwB,CAAC,oBAAoB,CAAC;gBAClD,UAAU;gBACV,wBAAwB,EAAE,KAAK;aAChC,CAAC,CAAC;YAEL;;;;eAIG;YACH,MAAM,iBAAiB,GACrB,MAAM,QAAQ,CAAC,+BAA+B,CAAC;gBAC7C,SAAS;gBACT,iBAAiB,EAAE,UAAU;gBAC7B,YAAY,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;qBAChD,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;oBAC5B,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtB,CAAC,CAAC;qBACD,MAAM,CAAC,OAAO,CAAC;gBAClB,UAAU,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC;qBAC5C,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;oBAC5B,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtB,CAAC,CAAC;qBACD,MAAM,CAAC,OAAO,CAAC;aACnB,CAAC,CAAC;YAEL,cAAc;gBACZ,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,GAAG,iBAAiB,CAAC;QAC/D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf;;;eAGG;YACH,MAAM,oBAAoB,CAAC,aAAa,CAAC;gBACvC,OAAO;gBACP,YAAY;gBACZ,YAAY,EAAE,qCACZ,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE;gBACF,WAAW,EAAE,KAAK;aACnB,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,qBAAqB,CAAC,UAAU,CAAC;YACrC,OAAO;YACP,SAAS;YACT,YAAY;YACZ,OAAO,EAAE;gBACP,gEAAgE;gBAChE,OAAO,EAAE,iCAAiC;gBAC1C,cAAc;gBACd,YAAY,EAAE,KAAK,EAAE,QAIpB,EAAiB,EAAE;oBAClB,MAAM,mBAAmB,CAAC,sBAAsB,CAAC;wBAC/C,UAAU;wBACV,SAAS;wBACT,qBAAqB,EAAE,qBAAqB,CAAC,SAAS;wBACtD,YAAY,EAAE,OAAO;wBACrB,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB;wBAC7C;;;;;2BAKG;wBACH,qBAAqB,EAAE;4BACrB,yBAAyB,EACvB,kBAAkB,CAAC,+BAA+B,CAChD,QAAQ,CAAC,UAAU,CACpB;yBACJ;qBACF,CAAC,CAAC;oBAEH;;;;;;;;;;uBAUG;oBACH,IAAI,CAAC;wBACH,MAAM,IAAI,GAAyB,IAAI,oBAAoB,EAAE,CAAC;wBAC9D,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;wBAC7B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;wBAC3B,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,gBAAgB,CAAC;wBACtC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;wBAE5B,MAAM,2BAA2B,CAAC,MAAM,CAAC;4BACvC,IAAI,EAAE,IAAI;4BACV,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE;yBAC3C,CAAC,CAAC;oBACL,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,MAAM,CAAC,KAAK,CACV,qDAAqD,UAAU,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACvF,CAAC;oBACJ,CAAC;oBAED;;;;;uBAKG;oBACH,IAAI,CAAC;wBACH,MAAM,eAAe,CAAC,iCAAiC,CAAC;4BACtD,UAAU;yBACX,CAAC,CAAC;oBACL,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,MAAM,CAAC,KAAK,CACV,wDAAwD,UAAU,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CAC1F,CAAC;oBACJ,CAAC;oBAED;;;;;;;;;;uBAUG;oBACH,IACE,kBAAkB,CAAC,gCAAgC,CACjD,QAAQ,CAAC,UAAU,CACpB,EACD,CAAC;wBACD,MAAM,0BAA0B,CAAC,mCAAmC,CAClE;4BACE,SAAS;4BACT,UAAU;yBACX,CACF,CAAC;oBACJ,CAAC;gBACH,CAAC;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,6DAA6D;IACrD,MAAM,CAAC,oBAAoB,CACjC,WAAgC;;QAEhC,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,WAAW,CAAC;QAEhD,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,UAAU,QAAQ,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;QAEhD,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,gBAAgB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,aAAa,CAAA,MAAA,QAAQ,CAAC,gBAAgB,0CAAE,IAAI,KAAI,KAAK,EAAE,CAAC,CAAC;QACpE,KAAK,CAAC,IAAI,CACR,kBAAkB,CAAA,MAAA,QAAQ,CAAC,oBAAoB,0CAAE,IAAI,KAAI,KAAK,EAAE,CACjE,CAAC;QACF,KAAK,CAAC,IAAI,CACR,gBACE,QAAQ,CAAC,SAAS;YAChB,CAAC,CAAC,aAAa,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC5D,CAAC,CAAC,KACN,EAAE,CACH,CAAC;QAEF,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,KAAK,CAAC,IAAI,CACR,sBAAsB,QAAQ,CAAC,QAAQ;iBACpC,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;gBAC5B,OAAO,CAAC,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,KAAK,CAAC,IAAI,CACR,WAAW,QAAQ,CAAC,MAAM;iBACvB,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;gBAC5B,OAAO,CAAC,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,oCAAoC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,WAAW,GAAkB,CAAC,aAAa,IAAI,EAAE,CAAC;aACrD,KAAK,CAAC,CAAC,CAAC,CAAC;aACT,GAAG,CAAC,CAAC,IAAuB,EAAE,EAAE;YAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF;AArQqB;IADnB,WAAW,EAAE;;;;iEA0Bb;AASmB;IADnB,WAAW,EAAE;;;;+DA+Jb"}

package/build/dist/Server/Utils/AI/{Sentinel → SRE}/IncidentPostmortemRunner.js

@@ -11,13 +11,13 @@ import { Blue500 } from "../../../../Types/BrandColors";
 import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
 import IncidentService from "../../../Services/IncidentService";
 import IncidentFeedService from "../../../Services/IncidentFeedService";
-import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
+import AIInvestigationEngine from "./AIInvestigationEngine";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 /*
- * Sentinel — auto-draft postmortem on resolve (Phase 3, "close the loop").
+ * AI SRE — auto-draft postmortem on resolve (Phase 3, "close the loop").
  *
- * When an incident moves to a resolved state, Sentinel drafts a postmortem from
+ * When an incident moves to a resolved state, AI drafts a postmortem from
  * the incident timeline + telemetry (reusing the existing
  * IncidentService.generatePostmortemFromAI) and saves it to the incident for a
  * human to review and edit — turning a ~90-minute manual writeup into a review.
@@ -27,11 +27,11 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  * fire-and-forget: failures are logged, never surfaced to the resolve flow.
  */
 const MAX_FEED_PREVIEW_CHARS = 6000;
-export default class SentinelIncidentPostmortemRunner {
+export default class AIIncidentPostmortemRunner {
     static async draftPostmortemOnResolve(data) {
         const { incidentId, projectId } = data;
         try {
-            if (!(await SentinelInvestigationEngine.isEnabledForProject(projectId, "Incident"))) {
+            if (!(await AIInvestigationEngine.isEnabledForProject(projectId, "Incident"))) {
                 return;
             }
             const incident = await IncidentService.findOneById({
@@ -69,7 +69,7 @@ export default class SentinelIncidentPostmortemRunner {
                 projectId,
                 incidentFeedEventType: IncidentFeedEventType.PostmortemNote,
                 displayColor: Blue500,
-                feedInfoInMarkdown: `## 🧠 Sentinel — Draft Postmortem\n\nSentinel drafted a postmortem for incident #${incident.incidentNumber} from the incident timeline and telemetry. It has been saved on the incident for you to review and edit.`,
+                feedInfoInMarkdown: `## 🧠 AI — Draft Postmortem\n\nOneUptime AI drafted a postmortem for incident #${incident.incidentNumber} from the incident timeline and telemetry. It has been saved on the incident for you to review and edit.`,
                 moreInformationInMarkdown: draft.length > MAX_FEED_PREVIEW_CHARS
                     ? `${draft.substring(0, MAX_FEED_PREVIEW_CHARS)}\n\n…(truncated — the full draft is saved on the incident.)`
                     : draft,
@@ -77,10 +77,10 @@ export default class SentinelIncidentPostmortemRunner {
                     sendWorkspaceNotification: true,
                 },
             });
-            logger.debug(`Sentinel: drafted postmortem for incident ${incidentId.toString()}.`);
+            logger.debug(`AI: drafted postmortem for incident ${incidentId.toString()}.`);
         }
         catch (error) {
-            logger.error(`Sentinel: failed to draft postmortem for incident ${incidentId.toString()}: ${error}`);
+            logger.error(`AI: failed to draft postmortem for incident ${incidentId.toString()}: ${error}`);
         }
     }
 }
@@ -89,5 +89,5 @@ __decorate([
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
-], SentinelIncidentPostmortemRunner, "draftPostmortemOnResolve", null);
+], AIIncidentPostmortemRunner, "draftPostmortemOnResolve", null);
 //# sourceMappingURL=IncidentPostmortemRunner.js.map

package/build/dist/Server/Utils/AI/SRE/IncidentPostmortemRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IncidentPostmortemRunner.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/SRE/IncidentPostmortemRunner.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,EAAE,OAAO,EAAE,MAAM,+BAA+B,CAAC;AAExD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gDAAgD,CAAC;AACvF,OAAO,eAAe,MAAM,mCAAmC,CAAC;AAChE,OAAO,mBAAmB,MAAM,uCAAuC,CAAC;AACxE,OAAO,qBAAqB,MAAM,yBAAyB,CAAC;AAC5D,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAEtD;;;;;;;;;;;GAWG;AACH,MAAM,sBAAsB,GAAW,IAAI,CAAC;AAE5C,MAAM,CAAC,OAAO,OAAO,0BAA0B;IAEzB,AAAb,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,IAG5C;QACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEvC,IAAI,CAAC;YACH,IACE,CAAC,CAAC,MAAM,qBAAqB,CAAC,mBAAmB,CAC/C,SAAS,EACT,UAAU,CACX,CAAC,EACF,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,QAAQ,GAAoB,MAAM,eAAe,CAAC,WAAW,CAAC;gBAClE,EAAE,EAAE,UAAU;gBACd,MAAM,EAAE;oBACN,GAAG,EAAE,IAAI;oBACT,cAAc,EAAE,IAAI;oBACpB,cAAc,EAAE,IAAI;iBACrB;gBACD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO;YACT,CAAC;YAED,sEAAsE;YACtE,IACE,QAAQ,CAAC,cAAc;gBACvB,QAAQ,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EACzC,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,KAAK,GAAW,MAAM,eAAe,CAAC,wBAAwB,CAAC;gBACnE,UAAU;aACX,CAAC,CAAC;YAEH,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC5B,OAAO;YACT,CAAC;YAED,MAAM,eAAe,CAAC,aAAa,CAAC;gBAClC,EAAE,EAAE,UAAU;gBACd,IAAI,EAAE;oBACJ,cAAc,EAAE,KAAK;iBACtB;gBACD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;YAEH,MAAM,mBAAmB,CAAC,sBAAsB,CAAC;gBAC/C,UAAU;gBACV,SAAS;gBACT,qBAAqB,EAAE,qBAAqB,CAAC,cAAc;gBAC3D,YAAY,EAAE,OAAO;gBACrB,kBAAkB,EAAE,kFAAkF,QAAQ,CAAC,cAAc,0GAA0G;gBACvO,yBAAyB,EACvB,KAAK,CAAC,MAAM,GAAG,sBAAsB;oBACnC,CAAC,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,sBAAsB,CAAC,6DAA6D;oBAC5G,CAAC,CAAC,KAAK;gBACX,qBAAqB,EAAE;oBACrB,yBAAyB,EAAE,IAAI;iBAChC;aACF,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CACV,uCAAuC,UAAU,CAAC,QAAQ,EAAE,GAAG,CAChE,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,+CAA+C,UAAU,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACjF,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AA9EqB;IADnB,WAAW,EAAE;;;;gEA8Eb"}

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/ErrorLogSpikeDetector.js

@@ -0,0 +1,268 @@
+import ObjectID from "../../../../../../Types/ObjectID";
+import OneUptimeDate from "../../../../../../Types/Date";
+import AIInsightType from "../../../../../../Types/AI/AIInsightType";
+import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
+import LogAggregationService from "../../../../../Services/LogAggregationService";
+import ServiceService from "../../../../../Services/ServiceService";
+/*
+ * ErrorLogSpike — the project's Error/Fatal log volume in the last hour
+ * spiked well above its own average hourly rate over the prior 24 hours.
+ *
+ * Two-stage on purpose (cost): stage 1 is ONE project-wide severity
+ * histogram, which ClickHouse serves from the proj_severity_histogram
+ * projection because severityText and the minute bucket are the
+ * projection's own keys. Only when stage 1 finds a spike does stage 2 run
+ * the service-attribution query — a top-list grouped by primaryEntityId,
+ * which falls back to a base-table scan but is bounded to the single spike
+ * hour. Deterministic — no LLM anywhere in this detector.
+ */
+// The spike window, compared against the prior baseline window.
+export const ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES = 60;
+/*
+ * Baseline: the 24 hours before the spike window, averaged per hour.
+ * Self-baselining means chronically chatty projects don't alert on their
+ * normal error volume.
+ */
+export const ERROR_LOG_SPIKE_BASELINE_WINDOW_HOURS = 24;
+/*
+ * Absolute floor: below 100 errors/hour a "3x spike" can be 20→100 raw
+ * lines — routine flakiness, not an event worth a human's attention.
+ */
+export const ERROR_LOG_SPIKE_MIN_RECENT_COUNT = 100;
+/*
+ * Volume multiplier that counts as a spike. The baseline hourly average is
+ * floored at 1 (see evaluateSpike) so a silent project's first errors
+ * still register without producing infinite multipliers.
+ */
+export const ERROR_LOG_SPIKE_MIN_MULTIPLIER = 3;
+// A >= 10x spike is a step change, not a fluctuation — escalate to High.
+export const ERROR_LOG_SPIKE_HIGH_SEVERITY_MULTIPLIER = 10;
+/*
+ * Histogram bucket size. The projection is minute-grained, so any bucket
+ * size is projection-served; 5 minutes keeps the recent/prior split error
+ * bounded to one bucket (< 5 min of data) while returning only ~300 rows
+ * for the whole 25h window.
+ */
+export const ERROR_LOG_SPIKE_HISTOGRAM_BUCKET_MINUTES = 5;
+/*
+ * At most this many per-service insights per spike. More than 3 services
+ * spiking together is one platform event, not N service events — the
+ * project-wide evidence block carries the full attribution list anyway.
+ */
+export const ERROR_LOG_SPIKE_MAX_ATTRIBUTED_SERVICES = 3;
+// Severity levels that count as "errors" for this detector (OTel names).
+export const ERROR_LOG_SEVERITIES = ["Error", "Fatal"];
+/*
+ * A ClickHouse DateTime rendered with no zone designator: "YYYY-MM-DD
+ * hh:mm:ss" (the shape LogAggregationService.getHistogram reads out of the
+ * `bucket` column), optionally "T"-separated and/or fractional. Anchored on
+ * purpose — any string carrying a zone (…Z, …+05:30) must NOT match, because
+ * parseBucketTime appends a Z to whatever matches and a double-designator
+ * string parses to an Invalid Date (which splitHistogram would then skip,
+ * zeroing the bucket).
+ */
+const ZONE_LESS_DATETIME_REGEX = /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(\.\d+)?$/;
+export default class ErrorLogSpikeDetector {
+    constructor() {
+        this.insightType = AIInsightType.ErrorLogSpike;
+    }
+    /*
+     * ClickHouse serves DateTime columns as a zone-less wall clock in the
+     * column's own zone — UTC in every OneUptime deployment. `new Date()` reads
+     * that shape as the PROCESS's LOCAL time, so on a worker whose TZ is not UTC
+     * every bucket silently slides by the UTC offset and the recent/prior
+     * boundary below lands in the wrong place: positive offsets push the spike's
+     * buckets into the baseline (suppressing every spike while inflating the
+     * average it is compared against), negative offsets drag hours of baseline
+     * into the "last 60 minutes" (fabricating spikes whose counts cannot even
+     * reconcile with stage 2, which binds real Date params and queries the true
+     * hour). Normalizing to an explicit UTC instant makes the split depend only
+     * on the data, never on the worker's clock configuration.
+     *
+     * Zone-carrying strings are parsed verbatim — see ZONE_LESS_DATETIME_REGEX.
+     * Anything else stays an Invalid Date so splitHistogram's isNaN guard still
+     * skips it. Same idiom as the dashboard's EdgeDetailPanel.parseBucketStart.
+     */
+    static parseBucketTime(time) {
+        const trimmedTime = time.trim();
+        if (ZONE_LESS_DATETIME_REGEX.test(trimmedTime)) {
+            return new Date(`${trimmedTime.replace(" ", "T")}Z`);
+        }
+        return new Date(trimmedTime);
+    }
+    /*
+     * Pure split of one histogram into the recent window and the prior
+     * baseline window. A bucket belongs to "recent" when it STARTS at or
+     * after the boundary; the boundary-straddling bucket therefore counts as
+     * prior — a deliberate conservative bias (understates the spike by at
+     * most one bucket of data rather than inflating it).
+     */
+    static splitHistogram(buckets, recentWindowStart) {
+        let recentCount = 0;
+        let priorCount = 0;
+        for (const bucket of buckets) {
+            const bucketTime = ErrorLogSpikeDetector.parseBucketTime(bucket.time);
+            if (isNaN(bucketTime.getTime())) {
+                continue;
+            }
+            if (bucketTime.getTime() >= recentWindowStart.getTime()) {
+                recentCount += bucket.count;
+            }
+            else {
+                priorCount += bucket.count;
+            }
+        }
+        return { recentCount, priorCount };
+    }
+    /*
+     * Pure decision: recent-hour error count vs the prior 24h window count.
+     * Spike ⇔ recent >= ERROR_LOG_SPIKE_MIN_RECENT_COUNT AND
+     * recent >= ERROR_LOG_SPIKE_MIN_MULTIPLIER * max(prior/24, 1).
+     */
+    static evaluateSpike(recentCount, priorWindowCount) {
+        const baselineHourlyAverage = priorWindowCount / ERROR_LOG_SPIKE_BASELINE_WINDOW_HOURS;
+        const effectiveBaseline = Math.max(baselineHourlyAverage, 1);
+        const multiplier = recentCount / effectiveBaseline;
+        const isSpike = recentCount >= ERROR_LOG_SPIKE_MIN_RECENT_COUNT &&
+            multiplier >= ERROR_LOG_SPIKE_MIN_MULTIPLIER;
+        return {
+            isSpike,
+            multiplier,
+            severity: multiplier >= ERROR_LOG_SPIKE_HIGH_SEVERITY_MULTIPLIER
+                ? AIInsightSeverity.High
+                : AIInsightSeverity.Medium,
+            recentCount,
+            baselineHourlyAverage,
+        };
+    }
+    /*
+     * Stable dedupe key: per attributed service, or "project" for the
+     * unattributed project-wide spike.
+     */
+    static buildFingerprint(serviceId) {
+        return `error-log-spike:${serviceId || "project"}`;
+    }
+    async detect(context) {
+        const recentWindowStart = OneUptimeDate.addRemoveMinutes(context.now, -1 * ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES);
+        const baselineWindowStart = OneUptimeDate.addRemoveHours(recentWindowStart, -1 * ERROR_LOG_SPIKE_BASELINE_WINDOW_HOURS);
+        /*
+         * Stage 1 — projection-fast project-wide severity histogram over the
+         * whole 25h (baseline + spike) range. severityTexts stays on the
+         * projection; adding any service/entity filter here would silently
+         * force a base-table scan (LogAggregationService.appendCommonFilters).
+         */
+        const buckets = await LogAggregationService.getHistogram({
+            projectId: context.projectId,
+            startTime: baselineWindowStart,
+            endTime: context.now,
+            bucketSizeInMinutes: ERROR_LOG_SPIKE_HISTOGRAM_BUCKET_MINUTES,
+            severityTexts: ERROR_LOG_SEVERITIES,
+        });
+        const { recentCount, priorCount } = ErrorLogSpikeDetector.splitHistogram(buckets, recentWindowStart);
+        const decision = ErrorLogSpikeDetector.evaluateSpike(recentCount, priorCount);
+        if (!decision.isSpike) {
+            return [];
+        }
+        /*
+         * Stage 2 — service attribution, only now that a spike exists. This
+         * top-list groups by primaryEntityId, which rejects the projection and
+         * scans the base table — acceptable because the scan is bounded to the
+         * single spike hour.
+         */
+        const topItems = await LogAggregationService.getAnalyticsTopList({
+            projectId: context.projectId,
+            startTime: recentWindowStart,
+            endTime: context.now,
+            bucketSizeInMinutes: ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES,
+            chartType: "toplist",
+            groupBy: ["primaryEntityId"],
+            aggregation: "count",
+            severityTexts: ERROR_LOG_SEVERITIES,
+            limit: ERROR_LOG_SPIKE_MAX_ATTRIBUTED_SERVICES,
+        });
+        /*
+         * Resolve entity ids to Service names. Non-Service entities (hosts,
+         * clusters) resolve to null and keep their raw id as the label — the
+         * evidence must stand alone even when attribution is imperfect.
+         */
+        const topServices = [];
+        for (const item of topItems) {
+            const service = await ServiceService.findOneById({
+                id: new ObjectID(item.value),
+                select: { name: true },
+                props: { isRoot: true },
+            });
+            topServices.push({
+                serviceId: item.value,
+                serviceName: (service === null || service === void 0 ? void 0 : service.name) || item.value,
+                isService: Boolean(service === null || service === void 0 ? void 0 : service.name),
+                count: item.count,
+            });
+        }
+        const detailLines = [
+            "**Error-log spike detected**",
+            "",
+            `- Error/Fatal logs in the last ${ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES} minutes: ${decision.recentCount}`,
+            `- Baseline: ${decision.baselineHourlyAverage.toFixed(2)}/hour over the prior ${ERROR_LOG_SPIKE_BASELINE_WINDOW_HOURS}h (${priorCount} total)`,
+            `- Spike multiplier: ${decision.multiplier.toFixed(1)}x`,
+        ];
+        if (topServices.length > 0) {
+            detailLines.push("", "Top contributing services in the spike window:");
+            for (const top of topServices) {
+                detailLines.push(`- ${top.serviceName}: ${top.count} error logs`);
+            }
+        }
+        const detailMarkdown = detailLines.join("\n");
+        const evidenceTopServices = topServices.map((top) => {
+            return { serviceName: top.serviceName, count: top.count };
+        });
+        /*
+         * One insight per top contributing service so each can be routed and
+         * triaged in its own context; when attribution found nothing, a single
+         * project-wide insight still surfaces the event.
+         */
+        if (topServices.length === 0) {
+            return [
+                {
+                    insightType: AIInsightType.ErrorLogSpike,
+                    fingerprint: ErrorLogSpikeDetector.buildFingerprint(undefined),
+                    title: `Error-log spike: ${decision.multiplier.toFixed(1)}x normal volume across the project`,
+                    detailMarkdown,
+                    severity: decision.severity,
+                    evidence: {
+                        logSpike: {
+                            recentErrorCount: decision.recentCount,
+                            baselineHourlyAverage: decision.baselineHourlyAverage,
+                            spikeMultiplier: decision.multiplier,
+                            windowMinutes: ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES,
+                            topServices: evidenceTopServices,
+                        },
+                    },
+                },
+            ];
+        }
+        return topServices.map((top) => {
+            return {
+                insightType: AIInsightType.ErrorLogSpike,
+                fingerprint: ErrorLogSpikeDetector.buildFingerprint(top.serviceId),
+                title: `Error-log spike: ${decision.multiplier.toFixed(1)}x normal volume in ${top.serviceName}`,
+                detailMarkdown,
+                severity: decision.severity,
+                serviceName: top.serviceName,
+                telemetryServiceId: top.isService
+                    ? new ObjectID(top.serviceId)
+                    : undefined,
+                evidence: {
+                    logSpike: {
+                        recentErrorCount: decision.recentCount,
+                        baselineHourlyAverage: decision.baselineHourlyAverage,
+                        spikeMultiplier: decision.multiplier,
+                        windowMinutes: ERROR_LOG_SPIKE_RECENT_WINDOW_MINUTES,
+                        topServices: evidenceTopServices,
+                    },
+                },
+            };
+        });
+    }
+}
+//# sourceMappingURL=ErrorLogSpikeDetector.js.map

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/ErrorLogSpikeDetector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ErrorLogSpikeDetector.js","sourceRoot":"","sources":["../../../../../../../../Server/Utils/AI/SRE/Insights/Detectors/ErrorLogSpikeDetector.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,kCAAkC,CAAC;AACxD,OAAO,aAAa,MAAM,8BAA8B,CAAC;AACzD,OAAO,aAAa,MAAM,0CAA0C,CAAC;AACrE,OAAO,iBAAiB,MAAM,8CAA8C,CAAC;AAE7E,OAAO,qBAGN,MAAM,+CAA+C,CAAC;AACvD,OAAO,cAAc,MAAM,wCAAwC,CAAC;AAOpE;;;;;;;;;;;GAWG;AAEH,gEAAgE;AAChE,MAAM,CAAC,MAAM,qCAAqC,GAAW,EAAE,CAAC;AAEhE;;;;GAIG;AACH,MAAM,CAAC,MAAM,qCAAqC,GAAW,EAAE,CAAC;AAEhE;;;GAGG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAW,GAAG,CAAC;AAE5D;;;;GAIG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAW,CAAC,CAAC;AAExD,yEAAyE;AACzE,MAAM,CAAC,MAAM,wCAAwC,GAAW,EAAE,CAAC;AAEnE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAW,CAAC,CAAC;AAElE;;;;GAIG;AACH,MAAM,CAAC,MAAM,uCAAuC,GAAW,CAAC,CAAC;AAEjE,yEAAyE;AACzE,MAAM,CAAC,MAAM,oBAAoB,GAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAEtE;;;;;;;;GAQG;AACH,MAAM,wBAAwB,GAC5B,kDAAkD,CAAC;AAYrD,MAAM,CAAC,OAAO,OAAO,qBAAqB;IAA1C;QACS,gBAAW,GAAkB,aAAa,CAAC,aAAa,CAAC;IA2QlE,CAAC;IAzQC;;;;;;;;;;;;;;;;OAgBG;IACI,MAAM,CAAC,eAAe,CAAC,IAAY;QACxC,MAAM,WAAW,GAAW,IAAI,CAAC,IAAI,EAAE,CAAC;QAExC,IAAI,wBAAwB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/C,OAAO,IAAI,IAAI,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QACvD,CAAC;QAED,OAAO,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;OAMG;IACI,MAAM,CAAC,cAAc,CAC1B,OAA+B,EAC/B,iBAAuB;QAEvB,IAAI,WAAW,GAAW,CAAC,CAAC;QAC5B,IAAI,UAAU,GAAW,CAAC,CAAC;QAE3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAS,qBAAqB,CAAC,eAAe,CAC5D,MAAM,CAAC,IAAI,CACZ,CAAC;YACF,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBAChC,SAAS;YACX,CAAC;YACD,IAAI,UAAU,CAAC,OAAO,EAAE,IAAI,iBAAiB,CAAC,OAAO,EAAE,EAAE,CAAC;gBACxD,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,aAAa,CACzB,WAAmB,EACnB,gBAAwB;QAExB,MAAM,qBAAqB,GACzB,gBAAgB,GAAG,qCAAqC,CAAC;QAC3D,MAAM,iBAAiB,GAAW,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;QACrE,MAAM,UAAU,GAAW,WAAW,GAAG,iBAAiB,CAAC;QAE3D,MAAM,OAAO,GACX,WAAW,IAAI,gCAAgC;YAC/C,UAAU,IAAI,8BAA8B,CAAC;QAE/C,OAAO;YACL,OAAO;YACP,UAAU;YACV,QAAQ,EACN,UAAU,IAAI,wCAAwC;gBACpD,CAAC,CAAC,iBAAiB,CAAC,IAAI;gBACxB,CAAC,CAAC,iBAAiB,CAAC,MAAM;YAC9B,WAAW;YACX,qBAAqB;SACtB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,gBAAgB,CAAC,SAA6B;QAC1D,OAAO,mBAAmB,SAAS,IAAI,SAAS,EAAE,CAAC;IACrD,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,OAA2B;QAE3B,MAAM,iBAAiB,GAAS,aAAa,CAAC,gBAAgB,CAC5D,OAAO,CAAC,GAAG,EACX,CAAC,CAAC,GAAG,qCAAqC,CAC3C,CAAC;QACF,MAAM,mBAAmB,GAAS,aAAa,CAAC,cAAc,CAC5D,iBAAiB,EACjB,CAAC,CAAC,GAAG,qCAAqC,CAC3C,CAAC;QAEF;;;;;WAKG;QACH,MAAM,OAAO,GACX,MAAM,qBAAqB,CAAC,YAAY,CAAC;YACvC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS,EAAE,mBAAmB;YAC9B,OAAO,EAAE,OAAO,CAAC,GAAG;YACpB,mBAAmB,EAAE,wCAAwC;YAC7D,aAAa,EAAE,oBAAoB;SACpC,CAAC,CAAC;QAEL,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,qBAAqB,CAAC,cAAc,CACtE,OAAO,EACP,iBAAiB,CAClB,CAAC;QAEF,MAAM,QAAQ,GAA0B,qBAAqB,CAAC,aAAa,CACzE,WAAW,EACX,UAAU,CACX,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED;;;;;WAKG;QACH,MAAM,QAAQ,GACZ,MAAM,qBAAqB,CAAC,mBAAmB,CAAC;YAC9C,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS,EAAE,iBAAiB;YAC5B,OAAO,EAAE,OAAO,CAAC,GAAG;YACpB,mBAAmB,EAAE,qCAAqC;YAC1D,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,CAAC,iBAAiB,CAAC;YAC5B,WAAW,EAAE,OAAO;YACpB,aAAa,EAAE,oBAAoB;YACnC,KAAK,EAAE,uCAAuC;SAC/C,CAAC,CAAC;QAEL;;;;WAIG;QACH,MAAM,WAAW,GAKZ,EAAE,CAAC;QAER,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAmB,MAAM,cAAc,CAAC,WAAW,CAAC;gBAC/D,EAAE,EAAE,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;gBAC5B,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;gBACtB,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;YACH,WAAW,CAAC,IAAI,CAAC;gBACf,SAAS,EAAE,IAAI,CAAC,KAAK;gBACrB,WAAW,EAAE,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,KAAI,IAAI,CAAC,KAAK;gBACxC,SAAS,EAAE,OAAO,CAAC,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,CAAC;gBACjC,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,WAAW,GAAkB;YACjC,8BAA8B;YAC9B,EAAE;YACF,kCAAkC,qCAAqC,aAAa,QAAQ,CAAC,WAAW,EAAE;YAC1G,eAAe,QAAQ,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,qCAAqC,MAAM,UAAU,SAAS;YAC9I,uBAAuB,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;SACzD,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,gDAAgD,CAAC,CAAC;YACvE,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC9B,WAAW,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,WAAW,KAAK,GAAG,CAAC,KAAK,aAAa,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QACD,MAAM,cAAc,GAAW,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEtD,MAAM,mBAAmB,GACvB,WAAW,CAAC,GAAG,CACb,CAAC,GAKA,EAA0C,EAAE;YAC3C,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;QAC5D,CAAC,CACF,CAAC;QAEJ;;;;WAIG;QACH,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACL;oBACE,WAAW,EAAE,aAAa,CAAC,aAAa;oBACxC,WAAW,EAAE,qBAAqB,CAAC,gBAAgB,CAAC,SAAS,CAAC;oBAC9D,KAAK,EAAE,oBAAoB,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,oCAAoC;oBAC7F,cAAc;oBACd,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,QAAQ,EAAE;wBACR,QAAQ,EAAE;4BACR,gBAAgB,EAAE,QAAQ,CAAC,WAAW;4BACtC,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;4BACrD,eAAe,EAAE,QAAQ,CAAC,UAAU;4BACpC,aAAa,EAAE,qCAAqC;4BACpD,WAAW,EAAE,mBAAmB;yBACjC;qBACF;iBACF;aACF,CAAC;QACJ,CAAC;QAED,OAAO,WAAW,CAAC,GAAG,CACpB,CAAC,GAKA,EAAoB,EAAE;YACrB,OAAO;gBACL,WAAW,EAAE,aAAa,CAAC,aAAa;gBACxC,WAAW,EAAE,qBAAqB,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC;gBAClE,KAAK,EAAE,oBAAoB,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,GAAG,CAAC,WAAW,EAAE;gBAChG,cAAc;gBACd,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,kBAAkB,EAAE,GAAG,CAAC,SAAS;oBAC/B,CAAC,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC;oBAC7B,CAAC,CAAC,SAAS;gBACb,QAAQ,EAAE;oBACR,QAAQ,EAAE;wBACR,gBAAgB,EAAE,QAAQ,CAAC,WAAW;wBACtC,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;wBACrD,eAAe,EAAE,QAAQ,CAAC,UAAU;wBACpC,aAAa,EAAE,qCAAqC;wBACpD,WAAW,EAAE,mBAAmB;qBACjC;iBACF;aACF,CAAC;QACJ,CAAC,CACF,CAAC;IACJ,CAAC;CACF"}