@oneuptime/common 11.4.1 → 11.5.0

package/build/dist/Server/Utils/AI/Sentinel/IncidentInvestigationRunner.js

@@ -10,9 +10,12 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 import OneUptimeDate from "../../../../Types/Date";
 import { Blue500 } from "../../../../Types/BrandColors";
 import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
+import IncidentInternalNote from "../../../../Models/DatabaseModels/IncidentInternalNote";
 import IncidentFeedService from "../../../Services/IncidentFeedService";
+import IncidentInternalNoteService from "../../../Services/IncidentInternalNoteService";
 import IncidentAIContextBuilder from "../IncidentAIContextBuilder";
 import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
+import SentinelInvestigationQueue from "./InvestigationQueue";
 import SentinelMemory from "./SentinelMemory";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
@@ -21,18 +24,17 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  *
  * When a new incident is declared, wakes Sentinel to investigate it (read-only)
  * and post a cited root-cause analysis into the incident timeline + Slack/Teams
- * — before the on-call engineer has finished reading the page. The heavy lifting
- * (run lifecycle, the agent loop, confidence gating) lives in the shared
- * SentinelInvestigationEngine; this file only assembles incident context and
- * posts the result to the incident's feed.
- *
- * Built on the fire-and-forget async pattern already used throughout
- * IncidentService.onCreateSuccess. See Internal/Roadmap/AISentinelExecution.md.
+ * — before the on-call engineer has finished reading the page. The trigger only
+ * records durable intent (a Queued AIRun via SentinelInvestigationQueue), so a
+ * pod restart can no longer orphan an investigation; the heavy lifting (run
+ * lifecycle, the agent loop, confidence gating) lives in the shared
+ * SentinelInvestigationEngine. See Internal/Roadmap/AISentinelExecution.md.
  */
 export default class SentinelIncidentInvestigationRunner {
     /*
      * Entry point called (fire-and-forget) from IncidentService.onCreateSuccess.
-     * Never throws — all failures are logged and recorded on the AIRun.
+     * Cheap: gates + a Queued AIRun row; execution happens via the queue.
+     * Never throws — all failures are logged.
      */
     static async investigateNewIncident(data) {
         const { incidentId, projectId } = data;
@@ -40,6 +42,25 @@ export default class SentinelIncidentInvestigationRunner {
             if (!(await SentinelInvestigationEngine.isEnabledForProject(projectId, "Incident"))) {
                 return;
             }
+            await SentinelInvestigationQueue.enqueue({
+                projectId,
+                subjectIncidentId: incidentId,
+            });
+        }
+        catch (error) {
+            logger.error(`Sentinel: unexpected error enqueueing investigation for incident ${incidentId.toString()}: ${error}`);
+        }
+    }
+    /*
+     * Execute a claimed run: assemble incident context and hand it to the
+     * engine. Called by SentinelInvestigationQueue after a successful claim.
+     * Never throws — failures are handed to the queue's retry policy so a
+     * claimed run is always finalized or requeued.
+     */
+    static async executeInvestigation(data) {
+        const { aiRunId, projectId, incidentId, attemptCount } = data;
+        let contextSummary;
+        try {
             const contextData = await IncidentAIContextBuilder.buildIncidentContext({
                 incidentId,
                 includeWorkspaceMessages: false,
@@ -63,11 +84,29 @@ export default class SentinelIncidentInvestigationRunner {
                 })
                     .filter(Boolean),
             });
-            await SentinelInvestigationEngine.investigate({
-                projectId,
+            contextSummary =
+                this.buildIncidentSummary(contextData) + priorCasesContext;
+        }
+        catch (error) {
+            /*
+             * Context assembly failed — the run is claimed, so hand it to the
+             * retry policy rather than leaving it Running until the sweeper.
+             */
+            await SentinelInvestigationQueue.failOrRequeue({
+                aiRunId,
+                attemptCount,
+                errorMessage: `Failed to build incident context: ${error instanceof Error ? error.message : String(error)}`,
+                isPermanent: false,
+            });
+            return;
+        }
+        await SentinelInvestigationEngine.executeRun({
+            aiRunId,
+            projectId,
+            attemptCount,
+            request: {
                 feature: "Sentinel Incident Investigation",
-                subjectIncidentId: incidentId,
-                contextSummary: this.buildIncidentSummary(contextData) + priorCasesContext,
+                contextSummary,
                 postAnalysis: async (postData) => {
                     await IncidentFeedService.createIncidentFeedItem({
                         incidentId,
@@ -83,12 +122,34 @@ export default class SentinelIncidentInvestigationRunner {
                             sendWorkspaceNotification: postData.isConfident,
                         },
                     });
+                    /*
+                     * Also file the RCA as an incident internal note, where responders
+                     * collaborate. The RootCause feed item above, with its quiet-mode
+                     * gating, must stay the single notification source, so:
+                     * ignoreHooks skips the note service's "posted private note" feed
+                     * item + unconditional workspace ping (workflow triggers and
+                     * realtime updates still fire), and isOwnerNotified stops the
+                     * note-posted owner-notification cron from paging owners about it.
+                     * Best-effort: a note failure must not fail the run after the feed
+                     * item already posted.
+                     */
+                    try {
+                        const note = new IncidentInternalNote();
+                        note.incidentId = incidentId;
+                        note.projectId = projectId;
+                        note.note = postData.analysisMarkdown;
+                        note.isOwnerNotified = true;
+                        await IncidentInternalNoteService.create({
+                            data: note,
+                            props: { isRoot: true, ignoreHooks: true },
+                        });
+                    }
+                    catch (error) {
+                        logger.error(`Sentinel: failed to post RCA internal note for incident ${incidentId.toString()}: ${error}`);
+                    }
                 },
-            });
-        }
-        catch (error) {
-            logger.error(`Sentinel: unexpected error investigating incident ${incidentId.toString()}: ${error}`);
-        }
+            },
+        });
     }
     // Build a compact incident record to seed the investigation.
     static buildIncidentSummary(contextData) {
@@ -144,4 +205,10 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
 ], SentinelIncidentInvestigationRunner, "investigateNewIncident", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SentinelIncidentInvestigationRunner, "executeInvestigation", null);
 //# sourceMappingURL=IncidentInvestigationRunner.js.map

package/build/dist/Server/Utils/AI/Sentinel/IncidentInvestigationRunner.js.map

@@ -1 +1 @@
-{"version":3,"file":"IncidentInvestigationRunner.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Sentinel/IncidentInvestigationRunner.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,aAAa,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,+BAA+B,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gDAAgD,CAAC;AACvF,OAAO,mBAAmB,MAAM,uCAAuC,CAAC;AACxE,OAAO,wBAEN,MAAM,6BAA6B,CAAC;AACrC,OAAO,2BAA2B,MAAM,+BAA+B,CAAC;AACxE,OAAO,cAAc,MAAM,kBAAkB,CAAC;AAE9C,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAEtD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,OAAO,OAAO,mCAAmC;IACtD;;;OAGG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAG1C;QACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEvC,IAAI,CAAC;YACH,IACE,CAAC,CAAC,MAAM,2BAA2B,CAAC,mBAAmB,CACrD,SAAS,EACT,UAAU,CACX,CAAC,EACF,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,WAAW,GACf,MAAM,wBAAwB,CAAC,oBAAoB,CAAC;gBAClD,UAAU;gBACV,wBAAwB,EAAE,KAAK;aAChC,CAAC,CAAC;YAEL;;;;eAIG;YACH,MAAM,iBAAiB,GACrB,MAAM,cAAc,CAAC,+BAA+B,CAAC;gBACnD,SAAS;gBACT,iBAAiB,EAAE,UAAU;gBAC7B,YAAY,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;qBAChD,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;oBAC5B,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtB,CAAC,CAAC;qBACD,MAAM,CAAC,OAAO,CAAC;gBAClB,UAAU,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC;qBAC5C,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;oBAC5B,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtB,CAAC,CAAC;qBACD,MAAM,CAAC,OAAO,CAAC;aACnB,CAAC,CAAC;YAEL,MAAM,2BAA2B,CAAC,WAAW,CAAC;gBAC5C,SAAS;gBACT,OAAO,EAAE,iCAAiC;gBAC1C,iBAAiB,EAAE,UAAU;gBAC7B,cAAc,EACZ,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,GAAG,iBAAiB;gBAC5D,YAAY,EAAE,KAAK,EAAE,QAIpB,EAAiB,EAAE;oBAClB,MAAM,mBAAmB,CAAC,sBAAsB,CAAC;wBAC/C,UAAU;wBACV,SAAS;wBACT,qBAAqB,EAAE,qBAAqB,CAAC,SAAS;wBACtD,YAAY,EAAE,OAAO;wBACrB,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB;wBAC7C;;;2BAGG;wBACH,qBAAqB,EAAE;4BACrB,yBAAyB,EAAE,QAAQ,CAAC,WAAW;yBAChD;qBACF,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,qDAAqD,UAAU,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,6DAA6D;IACrD,MAAM,CAAC,oBAAoB,CACjC,WAAgC;;QAEhC,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,WAAW,CAAC;QAEhD,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,UAAU,QAAQ,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;QAEhD,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,gBAAgB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,aAAa,CAAA,MAAA,QAAQ,CAAC,gBAAgB,0CAAE,IAAI,KAAI,KAAK,EAAE,CAAC,CAAC;QACpE,KAAK,CAAC,IAAI,CACR,kBAAkB,CAAA,MAAA,QAAQ,CAAC,oBAAoB,0CAAE,IAAI,KAAI,KAAK,EAAE,CACjE,CAAC;QACF,KAAK,CAAC,IAAI,CACR,gBACE,QAAQ,CAAC,SAAS;YAChB,CAAC,CAAC,aAAa,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC5D,CAAC,CAAC,KACN,EAAE,CACH,CAAC;QAEF,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,KAAK,CAAC,IAAI,CACR,sBAAsB,QAAQ,CAAC,QAAQ;iBACpC,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;gBAC5B,OAAO,CAAC,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,KAAK,CAAC,IAAI,CACR,WAAW,QAAQ,CAAC,MAAM;iBACvB,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;gBAC5B,OAAO,CAAC,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,oCAAoC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,WAAW,GAAkB,CAAC,aAAa,IAAI,EAAE,CAAC;aACrD,KAAK,CAAC,CAAC,CAAC,CAAC;aACT,GAAG,CAAC,CAAC,IAAuB,EAAE,EAAE;YAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF;AAhJqB;IADnB,WAAW,EAAE;;;;uEA4Eb"}
+{"version":3,"file":"IncidentInvestigationRunner.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Sentinel/IncidentInvestigationRunner.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,aAAa,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,+BAA+B,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gDAAgD,CAAC;AACvF,OAAO,oBAAoB,MAAM,wDAAwD,CAAC;AAC1F,OAAO,mBAAmB,MAAM,uCAAuC,CAAC;AACxE,OAAO,2BAA2B,MAAM,+CAA+C,CAAC;AACxF,OAAO,wBAEN,MAAM,6BAA6B,CAAC;AACrC,OAAO,2BAA2B,MAAM,+BAA+B,CAAC;AACxE,OAAO,0BAA0B,MAAM,sBAAsB,CAAC;AAC9D,OAAO,cAAc,MAAM,kBAAkB,CAAC;AAE9C,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAEtD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,OAAO,OAAO,mCAAmC;IACtD;;;;OAIG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAG1C;QACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEvC,IAAI,CAAC;YACH,IACE,CAAC,CAAC,MAAM,2BAA2B,CAAC,mBAAmB,CACrD,SAAS,EACT,UAAU,CACX,CAAC,EACF,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,0BAA0B,CAAC,OAAO,CAAC;gBACvC,SAAS;gBACT,iBAAiB,EAAE,UAAU;aAC9B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,oEAAoE,UAAU,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACtG,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAKxC;QACC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;QAE9D,IAAI,cAAsB,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,WAAW,GACf,MAAM,wBAAwB,CAAC,oBAAoB,CAAC;gBAClD,UAAU;gBACV,wBAAwB,EAAE,KAAK;aAChC,CAAC,CAAC;YAEL;;;;eAIG;YACH,MAAM,iBAAiB,GACrB,MAAM,cAAc,CAAC,+BAA+B,CAAC;gBACnD,SAAS;gBACT,iBAAiB,EAAE,UAAU;gBAC7B,YAAY,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;qBAChD,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;oBAC5B,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtB,CAAC,CAAC;qBACD,MAAM,CAAC,OAAO,CAAC;gBAClB,UAAU,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC;qBAC5C,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;oBAC5B,OAAO,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtB,CAAC,CAAC;qBACD,MAAM,CAAC,OAAO,CAAC;aACnB,CAAC,CAAC;YAEL,cAAc;gBACZ,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,GAAG,iBAAiB,CAAC;QAC/D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf;;;eAGG;YACH,MAAM,0BAA0B,CAAC,aAAa,CAAC;gBAC7C,OAAO;gBACP,YAAY;gBACZ,YAAY,EAAE,qCACZ,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE;gBACF,WAAW,EAAE,KAAK;aACnB,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,2BAA2B,CAAC,UAAU,CAAC;YAC3C,OAAO;YACP,SAAS;YACT,YAAY;YACZ,OAAO,EAAE;gBACP,OAAO,EAAE,iCAAiC;gBAC1C,cAAc;gBACd,YAAY,EAAE,KAAK,EAAE,QAIpB,EAAiB,EAAE;oBAClB,MAAM,mBAAmB,CAAC,sBAAsB,CAAC;wBAC/C,UAAU;wBACV,SAAS;wBACT,qBAAqB,EAAE,qBAAqB,CAAC,SAAS;wBACtD,YAAY,EAAE,OAAO;wBACrB,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB;wBAC7C;;;2BAGG;wBACH,qBAAqB,EAAE;4BACrB,yBAAyB,EAAE,QAAQ,CAAC,WAAW;yBAChD;qBACF,CAAC,CAAC;oBAEH;;;;;;;;;;uBAUG;oBACH,IAAI,CAAC;wBACH,MAAM,IAAI,GAAyB,IAAI,oBAAoB,EAAE,CAAC;wBAC9D,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;wBAC7B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;wBAC3B,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,gBAAgB,CAAC;wBACtC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;wBAE5B,MAAM,2BAA2B,CAAC,MAAM,CAAC;4BACvC,IAAI,EAAE,IAAI;4BACV,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE;yBAC3C,CAAC,CAAC;oBACL,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,MAAM,CAAC,KAAK,CACV,2DAA2D,UAAU,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CAC7F,CAAC;oBACJ,CAAC;gBACH,CAAC;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,6DAA6D;IACrD,MAAM,CAAC,oBAAoB,CACjC,WAAgC;;QAEhC,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,WAAW,CAAC;QAEhD,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,UAAU,QAAQ,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;QAEhD,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,gBAAgB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,aAAa,CAAA,MAAA,QAAQ,CAAC,gBAAgB,0CAAE,IAAI,KAAI,KAAK,EAAE,CAAC,CAAC;QACpE,KAAK,CAAC,IAAI,CACR,kBAAkB,CAAA,MAAA,QAAQ,CAAC,oBAAoB,0CAAE,IAAI,KAAI,KAAK,EAAE,CACjE,CAAC;QACF,KAAK,CAAC,IAAI,CACR,gBACE,QAAQ,CAAC,SAAS;YAChB,CAAC,CAAC,aAAa,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC5D,CAAC,CAAC,KACN,EAAE,CACH,CAAC;QAEF,IAAI,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,KAAK,CAAC,IAAI,CACR,sBAAsB,QAAQ,CAAC,QAAQ;iBACpC,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;gBAC5B,OAAO,CAAC,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,KAAK,CAAC,IAAI,CACR,WAAW,QAAQ,CAAC,MAAM;iBACvB,GAAG,CAAC,CAAC,CAAoB,EAAE,EAAE;gBAC5B,OAAO,CAAC,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,oCAAoC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,WAAW,GAAkB,CAAC,aAAa,IAAI,EAAE,CAAC;aACrD,KAAK,CAAC,CAAC,CAAC,CAAC;aACT,GAAG,CAAC,CAAC,IAAuB,EAAE,EAAE;YAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF;AAvNqB;IADnB,WAAW,EAAE;;;;uEA0Bb;AASmB;IADnB,WAAW,EAAE;;;;qEAiHb"}

package/build/dist/Server/Utils/AI/Sentinel/InvestigationQueue.js

@@ -0,0 +1,428 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import OneUptimeDate from "../../../../Types/Date";
+import AIRunType from "../../../../Types/AI/AIRunType";
+import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import AIRun from "../../../../Models/DatabaseModels/AIRun";
+import AIRunService from "../../../Services/AIRunService";
+import ProjectService from "../../../Services/ProjectService";
+import AIService from "../../../Services/AIService";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import QueryHelper from "../../../Types/Database/QueryHelper";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+/*
+ * Sentinel — the durable investigation queue (Phase 2's first item; Q1
+ * decided as the DB-claim pattern on AIRun rows, no new infrastructure).
+ *
+ * The problem it replaces: investigations ran as detached in-process
+ * promises, so a pod restart orphaned them silently (Deviations log D2).
+ *
+ * The shape:
+ *   1. enqueue() records the durable intent as an AIRun in status Queued
+ *      BEFORE any expensive work, then immediately tries to process it
+ *      in-process — same latency as before when the pod stays alive.
+ *   2. Claims go through AIRunService.attemptStatusTransition — one
+ *      conditional UPDATE guarded on BOTH status=Queued and the expected
+ *      attemptCount — so the enqueueing pod and the poller can race and
+ *      exactly one wins, and a stale queue snapshot can never claim (or
+ *      re-number) a run that moved on. attemptCount can therefore never
+ *      exceed MAX_INVESTIGATION_ATTEMPTS.
+ *   3. A Workers poller claims whatever the inline path left behind
+ *      (pod died, cap was full, budget was exhausted) and expires runs
+ *      that sat queued past their usefulness window. The poller claims
+ *      sequentially (so the concurrency cap sees each claim) but executes
+ *      detached, keeping the every-minute tick fast.
+ *   4. Failed attempts requeue (transient errors and stale heartbeats)
+ *      until MAX_ATTEMPTS, then finalize as Error/Stale — G9's retry
+ *      policy. Permanent errors (bad configuration) never retry.
+ *
+ * Mid-run message-level checkpointing is deliberately NOT here: a retried
+ * investigation re-runs from the top, which is safe because investigations
+ * are read-only until the single postAnalysis at the end.
+ */
+// Initial attempt + one retry.
+export const MAX_INVESTIGATION_ATTEMPTS = 2;
+/*
+ * G4 cost guardrail: at most this many investigations may be Running per
+ * project at once (per-project override in
+ * Project.aiMaxConcurrentInvestigations). Enforced at CLAIM time, so a storm
+ * queues (bounded by dedupe + severity gates + this TTL) and drains at cap
+ * rate instead of being dropped.
+ */
+export const DEFAULT_MAX_CONCURRENT_INVESTIGATIONS = 3;
+/*
+ * Clamp bounds for the per-project override. Pausing has its own switches
+ * (the opt-in toggles, daily limit 0), so the floor is 1, not 0.
+ */
+const MIN_CONCURRENT_INVESTIGATIONS = 1;
+const MAX_CONCURRENT_INVESTIGATIONS = 25;
+/*
+ * A first-pass RCA is only useful while the incident is fresh. Queued runs
+ * older than this are expired rather than run late — this also caps queue
+ * growth when the daily budget blocks claiming for hours.
+ */
+export const QUEUE_TTL_MINUTES = 30;
+// How many queued runs one poller tick will try to claim.
+const POLLER_BATCH_SIZE = 10;
+export default class SentinelInvestigationQueue {
+    /*
+     * Record the durable intent to investigate, then try to process it
+     * immediately (detached — the poller is the safety net if this pod dies).
+     * Callers have already passed the subject-specific gates (opt-in,
+     * severity floor, dedupe window).
+     */
+    static async enqueue(data) {
+        const { projectId } = data;
+        /*
+         * Budget quiet-skip at enqueue: when the daily budget is already
+         * exhausted there is no point recording intent that the TTL would
+         * expire anyway. Fails cheap (skip) like all the cost gates.
+         */
+        try {
+            const budget = await AIService.getAutonomousDailyBudgetStatus(projectId);
+            if (budget.exhausted) {
+                logger.debug(`Sentinel: not enqueueing investigation for project ${projectId.toString()} — daily autonomous token budget exhausted (${budget.usedTokensToday} of ${budget.limitInTokens} tokens used today).`);
+                return;
+            }
+        }
+        catch (error) {
+            logger.error(`Sentinel: budget check failed, not enqueueing investigation: ${error}`);
+            return;
+        }
+        const run = new AIRun();
+        run.projectId = projectId;
+        run.runType = AIRunType.Investigation;
+        run.status = AIRunStatus.Queued;
+        if (data.subjectIncidentId) {
+            run.triggeredByIncidentId = data.subjectIncidentId;
+        }
+        if (data.subjectAlertId) {
+            run.triggeredByAlertId = data.subjectAlertId;
+        }
+        if (data.subjectMonitorId) {
+            run.monitorId = data.subjectMonitorId;
+        }
+        let createdRun;
+        try {
+            createdRun = await AIRunService.create({
+                data: run,
+                props: { isRoot: true },
+            });
+        }
+        catch (error) {
+            logger.error(`Sentinel: failed to enqueue investigation run: ${error}`);
+            return;
+        }
+        // Inline kick — preserves the 1-3 minute RCA latency on the happy path.
+        this.processRun({
+            id: createdRun.id,
+            projectId,
+            attemptCount: 0,
+            triggeredByIncidentId: data.subjectIncidentId,
+            triggeredByAlertId: data.subjectAlertId,
+        }).catch((error) => {
+            var _a;
+            logger.error(`Sentinel: inline processing of queued run ${(_a = createdRun.id) === null || _a === void 0 ? void 0 : _a.toString()} failed: ${error}`);
+        });
+    }
+    /*
+     * Claim a queued run and execute it to completion. Safe to call from
+     * multiple places concurrently — the claim is one conditional UPDATE, so
+     * exactly one caller wins. Leaves the run Queued (for the poller / TTL)
+     * when the concurrency cap is full or the budget is exhausted.
+     */
+    static async processRun(run) {
+        try {
+            if (!(await this.passesClaimGates(run))) {
+                return;
+            }
+            const attempt = await this.claim(run);
+            if (attempt === null) {
+                return;
+            }
+            await this.dispatch(run, attempt);
+        }
+        catch (error) {
+            logger.error(`Sentinel: failed to process queued run ${run.id.toString()}: ${error}`);
+        }
+    }
+    /*
+     * One poller tick: expire runs that queued past their usefulness window,
+     * then claim a batch of the rest. Claims happen sequentially — each claim
+     * sets Running immediately, so the next iteration's concurrency-cap count
+     * sees it — but execution is detached so the tick finishes in seconds
+     * instead of holding the job open for the length of the investigations.
+     * Called every minute from Workers; also the recovery path for runs
+     * orphaned by pod restarts.
+     */
+    static async processQueuedRuns() {
+        const expiryThreshold = OneUptimeDate.getSomeMinutesAgo(QUEUE_TTL_MINUTES);
+        const expiredRuns = await AIRunService.findBy({
+            query: {
+                runType: AIRunType.Investigation,
+                status: AIRunStatus.Queued,
+                createdAt: QueryHelper.lessThan(expiryThreshold),
+            },
+            select: { _id: true },
+            limit: 100,
+            skip: 0,
+            props: { isRoot: true },
+        });
+        for (const expired of expiredRuns) {
+            await AIRunService.attemptStatusTransition({
+                aiRunId: expired.id,
+                fromStatus: AIRunStatus.Queued,
+                set: {
+                    status: AIRunStatus.Cancelled,
+                    completedAt: OneUptimeDate.getCurrentDate(),
+                    errorMessage: `Expired in the investigation queue after ${QUEUE_TTL_MINUTES} minutes — a first-pass analysis this late would no longer be useful. The project may have been at its concurrency cap or daily token budget.`,
+                },
+            });
+        }
+        const queuedRuns = await AIRunService.findBy({
+            query: {
+                runType: AIRunType.Investigation,
+                status: AIRunStatus.Queued,
+            },
+            select: {
+                _id: true,
+                projectId: true,
+                attemptCount: true,
+                triggeredByIncidentId: true,
+                triggeredByAlertId: true,
+            },
+            sort: { createdAt: SortOrder.Ascending },
+            limit: POLLER_BATCH_SIZE,
+            skip: 0,
+            props: { isRoot: true },
+        });
+        for (const queued of queuedRuns) {
+            const ref = {
+                id: queued.id,
+                projectId: queued.projectId,
+                attemptCount: queued.attemptCount || 0,
+                triggeredByIncidentId: queued.triggeredByIncidentId,
+                triggeredByAlertId: queued.triggeredByAlertId,
+            };
+            try {
+                if (!(await this.passesClaimGates(ref))) {
+                    continue;
+                }
+                const attempt = await this.claim(ref);
+                if (attempt === null) {
+                    continue;
+                }
+                /*
+                 * Execute detached — the run is claimed (Running + heartbeat), so
+                 * the sweeper owns recovery if this pod dies mid-flight.
+                 */
+                this.dispatch(ref, attempt).catch((error) => {
+                    logger.error(`Sentinel: detached execution of run ${ref.id.toString()} failed: ${error}`);
+                });
+            }
+            catch (error) {
+                logger.error(`Sentinel: poller failed on queued run ${ref.id.toString()}: ${error}`);
+            }
+        }
+    }
+    /*
+     * Finalize a failed attempt: requeue transient failures while attempts
+     * remain, otherwise mark the run Error. The transition guards on Running
+     * so a run that already completed (e.g. only postAnalysis failed) is
+     * never clobbered or re-run.
+     */
+    static async failOrRequeue(data) {
+        const truncatedMessage = data.errorMessage.substring(0, 400);
+        if (!data.isPermanent && data.attemptCount < MAX_INVESTIGATION_ATTEMPTS) {
+            const requeued = await AIRunService.attemptStatusTransition({
+                aiRunId: data.aiRunId,
+                fromStatus: AIRunStatus.Running,
+                set: {
+                    status: AIRunStatus.Queued,
+                    errorMessage: `Attempt ${data.attemptCount} failed and the run was requeued: ${truncatedMessage}`,
+                },
+            });
+            if (requeued > 0) {
+                logger.debug(`Sentinel: requeued run ${data.aiRunId.toString()} after attempt ${data.attemptCount} failed.`);
+            }
+            return;
+        }
+        await AIRunService.attemptStatusTransition({
+            aiRunId: data.aiRunId,
+            fromStatus: AIRunStatus.Running,
+            set: {
+                status: AIRunStatus.Error,
+                completedAt: OneUptimeDate.getCurrentDate(),
+                errorMessage: truncatedMessage,
+            },
+        });
+    }
+    /*
+     * Called by the stale-run sweeper for an Investigation run whose
+     * heartbeat went silent (the pod running it died). Requeues while
+     * attempts remain; otherwise marks it Stale as before.
+     */
+    static async requeueOrMarkStale(run) {
+        if ((run.attemptCount || 0) < MAX_INVESTIGATION_ATTEMPTS) {
+            const requeued = await AIRunService.attemptStatusTransition({
+                aiRunId: run.id,
+                fromStatus: AIRunStatus.Running,
+                set: {
+                    status: AIRunStatus.Queued,
+                    errorMessage: `Attempt ${run.attemptCount} stopped reporting progress (the server processing it may have restarted) and the run was requeued.`,
+                },
+            });
+            if (requeued > 0) {
+                return "requeued";
+            }
+        }
+        await AIRunService.attemptStatusTransition({
+            aiRunId: run.id,
+            fromStatus: AIRunStatus.Running,
+            set: {
+                status: AIRunStatus.Stale,
+                completedAt: OneUptimeDate.getCurrentDate(),
+                errorMessage: "The run stopped reporting progress and was marked as stale after exhausting its retry attempts. The server processing it may have restarted.",
+            },
+        });
+        return "stale";
+    }
+    /*
+     * The claim-time cost gates: concurrency cap and daily budget. A run
+     * failing these stays Queued — the poller retries and the TTL expires
+     * what never fits. Fails cheap (skip) on gate errors.
+     */
+    static async passesClaimGates(run) {
+        var _a;
+        // Per-project cap override, defaulting to 3 and clamped to [1, 25].
+        const project = await ProjectService.findOneById({
+            id: run.projectId,
+            select: { aiMaxConcurrentInvestigations: true },
+            props: { isRoot: true },
+        });
+        const concurrencyCap = Math.min(MAX_CONCURRENT_INVESTIGATIONS, Math.max(MIN_CONCURRENT_INVESTIGATIONS, (_a = project === null || project === void 0 ? void 0 : project.aiMaxConcurrentInvestigations) !== null && _a !== void 0 ? _a : DEFAULT_MAX_CONCURRENT_INVESTIGATIONS));
+        const runningCount = (await AIRunService.countBy({
+            query: {
+                projectId: run.projectId,
+                runType: AIRunType.Investigation,
+                status: AIRunStatus.Running,
+            },
+            props: { isRoot: true },
+        })).toNumber();
+        if (runningCount >= concurrencyCap) {
+            logger.debug(`Sentinel: leaving run ${run.id.toString()} queued — ${runningCount} investigations already running (cap: ${concurrencyCap}).`);
+            return false;
+        }
+        const budget = await AIService.getAutonomousDailyBudgetStatus(run.projectId);
+        if (budget.exhausted) {
+            logger.debug(`Sentinel: leaving run ${run.id.toString()} queued — daily autonomous token budget exhausted.`);
+            return false;
+        }
+        return true;
+    }
+    /*
+     * The atomic claim: Queued -> Running, guarded on the attemptCount the
+     * caller observed, so a stale queue snapshot can neither double-claim
+     * nor reset the attempt numbering. Returns the claimed attempt number,
+     * or null when another actor won.
+     */
+    static async claim(run) {
+        const attempt = (run.attemptCount || 0) + 1;
+        const claimedCount = await AIRunService.attemptStatusTransition({
+            aiRunId: run.id,
+            fromStatus: AIRunStatus.Queued,
+            expectedAttemptCount: run.attemptCount || 0,
+            set: {
+                status: AIRunStatus.Running,
+                startedAt: OneUptimeDate.getCurrentDate(),
+                lastHeartbeatAt: OneUptimeDate.getCurrentDate(),
+                attemptCount: attempt,
+            },
+        });
+        if (claimedCount === 0) {
+            return null;
+        }
+        return attempt;
+    }
+    // Route a claimed run to its subject's executor.
+    static async dispatch(run, attempt) {
+        if (run.triggeredByIncidentId) {
+            /*
+             * Lazy require: the runners import this queue to enqueue, so a
+             * top-level import here would be circular at module-init time
+             * (same pattern as DatabaseService -> AuditLogService).
+             */
+            const incidentRunner = 
+            // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+            require("./IncidentInvestigationRunner").default;
+            await incidentRunner.executeInvestigation({
+                aiRunId: run.id,
+                projectId: run.projectId,
+                incidentId: run.triggeredByIncidentId,
+                attemptCount: attempt,
+            });
+            return;
+        }
+        if (run.triggeredByAlertId) {
+            const alertRunner = 
+            // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+            require("./AlertInvestigationRunner").default;
+            await alertRunner.executeInvestigation({
+                aiRunId: run.id,
+                projectId: run.projectId,
+                alertId: run.triggeredByAlertId,
+                attemptCount: attempt,
+            });
+            return;
+        }
+        // A queued investigation without a subject cannot be executed.
+        await AIRunService.attemptStatusTransition({
+            aiRunId: run.id,
+            fromStatus: AIRunStatus.Running,
+            set: {
+                status: AIRunStatus.Error,
+                completedAt: OneUptimeDate.getCurrentDate(),
+                errorMessage: "Queued investigation has no subject to investigate.",
+            },
+        });
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SentinelInvestigationQueue, "enqueue", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SentinelInvestigationQueue, "processRun", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], SentinelInvestigationQueue, "processQueuedRuns", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SentinelInvestigationQueue, "failOrRequeue", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SentinelInvestigationQueue, "requeueOrMarkStale", null);
+//# sourceMappingURL=InvestigationQueue.js.map

package/build/dist/Server/Utils/AI/Sentinel/InvestigationQueue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InvestigationQueue.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Sentinel/InvestigationQueue.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,aAAa,MAAM,wBAAwB,CAAC;AACnD,OAAO,SAAS,MAAM,gCAAgC,CAAC;AACvD,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAC3D,OAAO,KAAK,MAAM,yCAAyC,CAAC;AAE5D,OAAO,YAAY,MAAM,gCAAgC,CAAC;AAC1D,OAAO,cAAc,MAAM,kCAAkC,CAAC;AAC9D,OAAO,SAAqC,MAAM,6BAA6B,CAAC;AAChF,OAAO,SAAS,MAAM,0CAA0C,CAAC;AACjE,OAAO,WAAW,MAAM,qCAAqC,CAAC;AAC9D,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,+BAA+B;AAC/B,MAAM,CAAC,MAAM,0BAA0B,GAAW,CAAC,CAAC;AAEpD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,qCAAqC,GAAW,CAAC,CAAC;AAC/D;;;GAGG;AACH,MAAM,6BAA6B,GAAW,CAAC,CAAC;AAChD,MAAM,6BAA6B,GAAW,EAAE,CAAC;AAEjD;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAW,EAAE,CAAC;AAE5C,0DAA0D;AAC1D,MAAM,iBAAiB,GAAW,EAAE,CAAC;AAUrC,MAAM,CAAC,OAAO,OAAO,0BAA0B;IAC7C;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAK3B;QACC,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAE3B;;;;WAIG;QACH,IAAI,CAAC;YACH,MAAM,MAAM,GACV,MAAM,SAAS,CAAC,8BAA8B,CAAC,SAAS,CAAC,CAAC;YAE5D,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,MAAM,CAAC,KAAK,CACV,sDAAsD,SAAS,CAAC,QAAQ,EAAE,+CAA+C,MAAM,CAAC,eAAe,OAAO,MAAM,CAAC,aAAa,sBAAsB,CACjM,CAAC;gBACF,OAAO;YACT,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,gEAAgE,KAAK,EAAE,CACxE,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAU,IAAI,KAAK,EAAE,CAAC;QAC/B,GAAG,CAAC,SAAS,GAAG,SAAS,CAAC;QAC1B,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,aAAa,CAAC;QACtC,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;QAEhC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,GAAG,CAAC,qBAAqB,GAAG,IAAI,CAAC,iBAAiB,CAAC;QACrD,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,GAAG,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAc,CAAC;QAC/C,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACxC,CAAC;QAED,IAAI,UAAiB,CAAC;QACtB,IAAI,CAAC;YACH,UAAU,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC;gBACrC,IAAI,EAAE,GAAG;gBACT,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,kDAAkD,KAAK,EAAE,CAAC,CAAC;YACxE,OAAO;QACT,CAAC;QAED,wEAAwE;QACxE,IAAI,CAAC,UAAU,CAAC;YACd,EAAE,EAAE,UAAU,CAAC,EAAG;YAClB,SAAS;YACT,YAAY,EAAE,CAAC;YACf,qBAAqB,EAAE,IAAI,CAAC,iBAAiB;YAC7C,kBAAkB,EAAE,IAAI,CAAC,cAAc;SACxC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;;YACxB,MAAM,CAAC,KAAK,CACV,6CAA6C,MAAA,UAAU,CAAC,EAAE,0CAAE,QAAQ,EAAE,YAAY,KAAK,EAAE,CAC1F,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,GAAiB;QAC9C,IAAI,CAAC;YACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBACxC,OAAO;YACT,CAAC;YAED,MAAM,OAAO,GAAkB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAErD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,OAAO;YACT,CAAC;YAED,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,0CAA0C,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACxE,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,iBAAiB;QACnC,MAAM,eAAe,GACnB,aAAa,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;QAErD,MAAM,WAAW,GAAiB,MAAM,YAAY,CAAC,MAAM,CAAC;YAC1D,KAAK,EAAE;gBACL,OAAO,EAAE,SAAS,CAAC,aAAa;gBAChC,MAAM,EAAE,WAAW,CAAC,MAAM;gBAC1B,SAAS,EAAE,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC;aACjD;YACD,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;YACrB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,CAAC;YACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QAEH,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;YAClC,MAAM,YAAY,CAAC,uBAAuB,CAAC;gBACzC,OAAO,EAAE,OAAO,CAAC,EAAG;gBACpB,UAAU,EAAE,WAAW,CAAC,MAAM;gBAC9B,GAAG,EAAE;oBACH,MAAM,EAAE,WAAW,CAAC,SAAS;oBAC7B,WAAW,EAAE,aAAa,CAAC,cAAc,EAAE;oBAC3C,YAAY,EAAE,4CAA4C,iBAAiB,+IAA+I;iBAC3N;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,UAAU,GAAiB,MAAM,YAAY,CAAC,MAAM,CAAC;YACzD,KAAK,EAAE;gBACL,OAAO,EAAE,SAAS,CAAC,aAAa;gBAChC,MAAM,EAAE,WAAW,CAAC,MAAM;aAC3B;YACD,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI;gBACT,SAAS,EAAE,IAAI;gBACf,YAAY,EAAE,IAAI;gBAClB,qBAAqB,EAAE,IAAI;gBAC3B,kBAAkB,EAAE,IAAI;aACzB;YACD,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE;YACxC,KAAK,EAAE,iBAAiB;YACxB,IAAI,EAAE,CAAC;YACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QAEH,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,MAAM,GAAG,GAAiB;gBACxB,EAAE,EAAE,MAAM,CAAC,EAAG;gBACd,SAAS,EAAE,MAAM,CAAC,SAAU;gBAC5B,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,CAAC;gBACtC,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;gBACnD,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;aAC9C,CAAC;YAEF,IAAI,CAAC;gBACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;oBACxC,SAAS;gBACX,CAAC;gBAED,MAAM,OAAO,GAAkB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAErD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED;;;mBAGG;gBACH,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;oBACjD,MAAM,CAAC,KAAK,CACV,uCAAuC,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CACV,yCAAyC,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACvE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAKjC;QACC,MAAM,gBAAgB,GAAW,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAErE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,GAAG,0BAA0B,EAAE,CAAC;YACxE,MAAM,QAAQ,GAAW,MAAM,YAAY,CAAC,uBAAuB,CAAC;gBAClE,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,UAAU,EAAE,WAAW,CAAC,OAAO;gBAC/B,GAAG,EAAE;oBACH,MAAM,EAAE,WAAW,CAAC,MAAM;oBAC1B,YAAY,EAAE,WAAW,IAAI,CAAC,YAAY,qCAAqC,gBAAgB,EAAE;iBAClG;aACF,CAAC,CAAC;YAEH,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBACjB,MAAM,CAAC,KAAK,CACV,0BAA0B,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,kBAAkB,IAAI,CAAC,YAAY,UAAU,CAC/F,CAAC;YACJ,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,YAAY,CAAC,uBAAuB,CAAC;YACzC,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,WAAW,CAAC,OAAO;YAC/B,GAAG,EAAE;gBACH,MAAM,EAAE,WAAW,CAAC,KAAK;gBACzB,WAAW,EAAE,aAAa,CAAC,cAAc,EAAE;gBAC3C,YAAY,EAAE,gBAAgB;aAC/B;SACF,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAGtC;QACC,IAAI,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,0BAA0B,EAAE,CAAC;YACzD,MAAM,QAAQ,GAAW,MAAM,YAAY,CAAC,uBAAuB,CAAC;gBAClE,OAAO,EAAE,GAAG,CAAC,EAAE;gBACf,UAAU,EAAE,WAAW,CAAC,OAAO;gBAC/B,GAAG,EAAE;oBACH,MAAM,EAAE,WAAW,CAAC,MAAM;oBAC1B,YAAY,EAAE,WAAW,GAAG,CAAC,YAAY,qGAAqG;iBAC/I;aACF,CAAC,CAAC;YAEH,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBACjB,OAAO,UAAU,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,YAAY,CAAC,uBAAuB,CAAC;YACzC,OAAO,EAAE,GAAG,CAAC,EAAE;YACf,UAAU,EAAE,WAAW,CAAC,OAAO;YAC/B,GAAG,EAAE;gBACH,MAAM,EAAE,WAAW,CAAC,KAAK;gBACzB,WAAW,EAAE,aAAa,CAAC,cAAc,EAAE;gBAC3C,YAAY,EACV,8IAA8I;aACjJ;SACF,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,GAAiB;;QACrD,oEAAoE;QACpE,MAAM,OAAO,GAAmB,MAAM,cAAc,CAAC,WAAW,CAAC;YAC/D,EAAE,EAAE,GAAG,CAAC,SAAS;YACjB,MAAM,EAAE,EAAE,6BAA6B,EAAE,IAAI,EAAE;YAC/C,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QAEH,MAAM,cAAc,GAAW,IAAI,CAAC,GAAG,CACrC,6BAA6B,EAC7B,IAAI,CAAC,GAAG,CACN,6BAA6B,EAC7B,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,6BAA6B,mCACpC,qCAAqC,CACxC,CACF,CAAC;QAEF,MAAM,YAAY,GAAW,CAC3B,MAAM,YAAY,CAAC,OAAO,CAAC;YACzB,KAAK,EAAE;gBACL,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,OAAO,EAAE,SAAS,CAAC,aAAa;gBAChC,MAAM,EAAE,WAAW,CAAC,OAAO;aAC5B;YACD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CACH,CAAC,QAAQ,EAAE,CAAC;QAEb,IAAI,YAAY,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,CAAC,KAAK,CACV,yBAAyB,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,YAAY,yCAAyC,cAAc,IAAI,CAC/H,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,MAAM,GACV,MAAM,SAAS,CAAC,8BAA8B,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEhE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,MAAM,CAAC,KAAK,CACV,yBAAyB,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,oDAAoD,CAC/F,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACK,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAiB;QAC1C,MAAM,OAAO,GAAW,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAEpD,MAAM,YAAY,GAAW,MAAM,YAAY,CAAC,uBAAuB,CAAC;YACtE,OAAO,EAAE,GAAG,CAAC,EAAE;YACf,UAAU,EAAE,WAAW,CAAC,MAAM;YAC9B,oBAAoB,EAAE,GAAG,CAAC,YAAY,IAAI,CAAC;YAC3C,GAAG,EAAE;gBACH,MAAM,EAAE,WAAW,CAAC,OAAO;gBAC3B,SAAS,EAAE,aAAa,CAAC,cAAc,EAAE;gBACzC,eAAe,EAAE,aAAa,CAAC,cAAc,EAAE;gBAC/C,YAAY,EAAE,OAAO;aACtB;SACF,CAAC,CAAC;QAEH,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,iDAAiD;IACzC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAC3B,GAAiB,EACjB,OAAe;QAEf,IAAI,GAAG,CAAC,qBAAqB,EAAE,CAAC;YAC9B;;;;eAIG;YACH,MAAM,cAAc;YAClB,qGAAqG;YACrG,OAAO,CAAC,+BAA+B,CAAC,CAAC,OAAO,CAAC;YAEnD,MAAM,cAAc,CAAC,oBAAoB,CAAC;gBACxC,OAAO,EAAE,GAAG,CAAC,EAAE;gBACf,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,UAAU,EAAE,GAAG,CAAC,qBAAqB;gBACrC,YAAY,EAAE,OAAO;aACtB,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC3B,MAAM,WAAW;YACf,qGAAqG;YACrG,OAAO,CAAC,4BAA4B,CAAC,CAAC,OAAO,CAAC;YAEhD,MAAM,WAAW,CAAC,oBAAoB,CAAC;gBACrC,OAAO,EAAE,GAAG,CAAC,EAAE;gBACf,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,OAAO,EAAE,GAAG,CAAC,kBAAkB;gBAC/B,YAAY,EAAE,OAAO;aACtB,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,+DAA+D;QAC/D,MAAM,YAAY,CAAC,uBAAuB,CAAC;YACzC,OAAO,EAAE,GAAG,CAAC,EAAE;YACf,UAAU,EAAE,WAAW,CAAC,OAAO;YAC/B,GAAG,EAAE;gBACH,MAAM,EAAE,WAAW,CAAC,KAAK;gBACzB,WAAW,EAAE,aAAa,CAAC,cAAc,EAAE;gBAC3C,YAAY,EAAE,qDAAqD;aACpE;SACF,CAAC,CAAC;IACL,CAAC;CACF;AAnZqB;IADnB,WAAW,EAAE;;;;+CAqEb;AASmB;IADnB,WAAW,EAAE;;;;kDAmBb;AAYmB;IADnB,WAAW,EAAE;;;;yDAkFb;AASmB;IADnB,WAAW,EAAE;;;;qDAoCb;AAQmB;IADnB,WAAW,EAAE;;;;0DAgCb"}