@oneuptime/common 11.5.4 → 11.5.6

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/TraceLatencyRegressionDetector.js

@@ -0,0 +1,384 @@
+import ObjectID from "../../../../../../Types/ObjectID";
+import OneUptimeDate from "../../../../../../Types/Date";
+import SortOrder from "../../../../../../Types/BaseDatabase/SortOrder";
+import InBetween from "../../../../../../Types/BaseDatabase/InBetween";
+import AIInsightType from "../../../../../../Types/AI/AIInsightType";
+import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
+import SpanService from "../../../../../Services/SpanService";
+import ServiceService from "../../../../../Services/ServiceService";
+import TraceAggregationService from "../../../../../Services/TraceAggregationService";
+import SpanTreeAnalyzer from "../../../PerfEvidence/SpanTreeAnalyzer";
+import FixPerformanceTaskTrigger from "../../FixPerformanceTaskTrigger";
+import logger from "../../../../Logger";
+/*
+ * TraceLatencyRegression — a service whose recent p99 span latency
+ * regressed to a multiple of its own 24-hour baseline, with the evidence
+ * drilled down to span-tree findings from a representative slow trace.
+ *
+ * Stage 1 is two per-service latency-profile queries (recent hour vs the
+ * prior 24h) via TraceAggregationService.getServiceLatencyProfile, which is
+ * served by the Span proj_agg_by_service aggregate projection: this runs
+ * unattended for every opted-in project every 15 minutes, so it must read
+ * pre-aggregated minute rows, not 25 hours of raw spans. p99 (not p95) on
+ * purpose: quantile(0.99) is the only percentile state that projection
+ * stores, and quantile states of different levels do not merge. The drill
+ * (slowest operation → representative trace → span-tree analysis → code
+ * locations) runs ONLY for services that regressed, and the resulting
+ * findings are stored on the insight because ClickHouse span retention is
+ * short — the stored evidence must outlive the spans (FixPerformance
+ * taskContext precedent). Deterministic — no LLM anywhere in this detector.
+ */
+// The regression window, compared against the prior baseline window.
+export const LATENCY_RECENT_WINDOW_HOURS = 1;
+/*
+ * Baseline: the 24 hours before the regression window. Self-baselining
+ * per service — a chronically slow service does not alert on being its
+ * usual slow self.
+ */
+export const LATENCY_BASELINE_WINDOW_HOURS = 24;
+/*
+ * Absolute floor: a p99 under one second is not worth a human's attention
+ * even if it doubled — sub-second "regressions" are dominated by cache
+ * and GC noise.
+ */
+export const LATENCY_MIN_RECENT_P99_MS = 1000;
+// p99 multiplier over the service's own baseline that counts as regression.
+export const LATENCY_MIN_REGRESSION_MULTIPLIER = 2;
+// A >= 4x p99 regression is a step change — escalate to High.
+export const LATENCY_HIGH_SEVERITY_MULTIPLIER = 4;
+/*
+ * The baseline p99 is only trustworthy with enough prior samples; below
+ * this the "baseline" is a handful of requests and the multiplier is
+ * noise. Matches the cold-start posture of the metric baselines.
+ */
+export const LATENCY_MIN_PRIOR_SAMPLE_COUNT = 100;
+/*
+ * Per-service rows fetched per window. The profile query ranks by request
+ * volume, so this bounds the scan to the 25 busiest services — the ones a
+ * p99 regression actually hurts (and the prior-sample-count gate would
+ * reject quiet services anyway).
+ */
+export const LATENCY_SERVICE_LIMIT = 25;
+/*
+ * Span cap for the representative-trace fetch — mirrors the trace
+ * waterfall tools' cap; beyond this the analyzer's evidence quality
+ * degrades anyway.
+ */
+export const LATENCY_MAX_TRACE_SPANS = 500;
+/*
+ * The drill's top-list ranks operations by p99, and for a group of ONE span
+ * the p99 IS that span's own duration — so a single freak-slow request would
+ * take the top slot outright. Fetch a small candidate set instead of one row
+ * and let the sample floor below choose among them. Ten is enough: an
+ * operation that cannot beat ten rivals on p99 is not the story.
+ */
+export const LATENCY_TOP_OPERATION_CANDIDATES = 10;
+/*
+ * Minimum spans an operation must have in the recent window before its p99
+ * is treated as evidence — and, just as load-bearing, before its name is
+ * allowed into the fingerprint. The fingerprint embeds the operation, so a
+ * noisy winner does not merely mislabel one insight: it churns the dedupe
+ * key tick after tick, and each new key is a NEW insight row plus a NEW
+ * triage LLM run for one and the same ongoing regression. Deliberately far
+ * below LATENCY_MIN_PRIOR_SAMPLE_COUNT — this gate only asks "is this
+ * operation real traffic", not "is this baseline trustworthy".
+ */
+export const LATENCY_MIN_OPERATION_SAMPLE_COUNT = 20;
+export default class TraceLatencyRegressionDetector {
+    constructor() {
+        this.insightType = AIInsightType.TraceLatencyRegression;
+    }
+    /*
+     * Pure decision: regression ⇔ recent p99 >= LATENCY_MIN_RECENT_P99_MS
+     * AND prior window has >= LATENCY_MIN_PRIOR_SAMPLE_COUNT samples AND
+     * recent p99 >= LATENCY_MIN_REGRESSION_MULTIPLIER * prior p99 (> 0).
+     */
+    static evaluateRegression(input) {
+        const multiplier = input.priorP99Ms > 0 ? input.recentP99Ms / input.priorP99Ms : 0;
+        const isRegression = input.recentP99Ms >= LATENCY_MIN_RECENT_P99_MS &&
+            input.priorSampleCount >= LATENCY_MIN_PRIOR_SAMPLE_COUNT &&
+            input.priorP99Ms > 0 &&
+            multiplier >= LATENCY_MIN_REGRESSION_MULTIPLIER;
+        return {
+            isRegression,
+            multiplier,
+            severity: multiplier >= LATENCY_HIGH_SEVERITY_MULTIPLIER
+                ? AIInsightSeverity.High
+                : AIInsightSeverity.Medium,
+        };
+    }
+    /*
+     * Pure join of the two per-service profile query results on
+     * primaryEntityId. Services missing from the prior window join with a
+     * zero baseline (and are then rejected by the sample-count gate).
+     */
+    static joinServiceWindows(recentRows, priorRows) {
+        const priorByEntity = new Map();
+        for (const row of priorRows) {
+            if (row.primaryEntityId) {
+                priorByEntity.set(row.primaryEntityId, row);
+            }
+        }
+        const joined = [];
+        for (const row of recentRows) {
+            const entityId = row.primaryEntityId || "";
+            if (!entityId) {
+                continue;
+            }
+            const prior = priorByEntity.get(entityId);
+            joined.push({
+                primaryEntityId: entityId,
+                recentP99Ms: row.p99DurationMs,
+                recentSampleCount: row.count,
+                priorP99Ms: (prior === null || prior === void 0 ? void 0 : prior.p99DurationMs) || 0,
+                priorSampleCount: (prior === null || prior === void 0 ? void 0 : prior.count) || 0,
+            });
+        }
+        return joined;
+    }
+    /*
+     * Pure decision: the slowest operation whose p99 is backed by real
+     * traffic. Candidates below LATENCY_MIN_OPERATION_SAMPLE_COUNT are
+     * dropped outright — with n=1 the "p99" is just that one span, so an
+     * unfiltered ranking hands the top slot to whatever freak request the
+     * window happened to catch.
+     *
+     * Ranking is re-established here rather than trusting the query's row
+     * order: ClickHouse's ORDER BY val DESC is not a total order, and ties
+     * broken arbitrarily by the server would move the fingerprint (which
+     * embeds the operation name) from tick to tick. Ties fall to volume,
+     * then to the name — a total, deterministic order.
+     *
+     * Undefined when nothing clears the floor. The caller then emits
+     * service-level evidence under the stable service-scoped fingerprint,
+     * which is strictly better than naming an operation we do not believe.
+     */
+    static pickSlowestOperation(items) {
+        var _a;
+        const credible = items.filter((item) => {
+            return (item.value.length > 0 &&
+                item.count >= LATENCY_MIN_OPERATION_SAMPLE_COUNT);
+        });
+        credible.sort((a, b) => {
+            if (b.metricValue !== a.metricValue) {
+                return b.metricValue - a.metricValue;
+            }
+            if (b.count !== a.count) {
+                return b.count - a.count;
+            }
+            return a.value.localeCompare(b.value);
+        });
+        return (_a = credible[0]) === null || _a === void 0 ? void 0 : _a.value;
+    }
+    /*
+     * Nanoseconds → milliseconds conversion into the analyzer's input shape
+     * (the AIInvestigationAPI conversion, kept local because the API file
+     * does not export it). Pure — the ns→ms arithmetic is unit-tested.
+     */
+    static toAnalyzableSpans(spans) {
+        return spans.map((span) => {
+            var _a, _b;
+            const attributes = {};
+            for (const [key, value] of Object.entries(span.attributes || {})) {
+                if (value !== null && value !== undefined) {
+                    attributes[key] = String(value);
+                }
+            }
+            return {
+                spanId: ((_a = span.spanId) === null || _a === void 0 ? void 0 : _a.toString()) || "",
+                parentSpanId: ((_b = span.parentSpanId) === null || _b === void 0 ? void 0 : _b.toString()) || undefined,
+                name: span.name || "",
+                startMs: Number(span.startTimeUnixNano) / 1000000,
+                endMs: Number(span.endTimeUnixNano) / 1000000,
+                durationMs: Number(span.durationUnixNano) / 1000000,
+                attributes,
+            };
+        });
+    }
+    /*
+     * Stable dedupe key: service + NORMALIZED operation name (ids/digits
+     * collapsed), so the same regressing endpoint refreshes one insight no
+     * matter which concrete URL the representative trace hit.
+     */
+    static buildFingerprint(serviceEntityId, operationName) {
+        return `latency:${serviceEntityId}:${SpanTreeAnalyzer.normalizeSpanName(operationName || "")}`;
+    }
+    async detect(context) {
+        const recentWindowStart = OneUptimeDate.addRemoveHours(context.now, -1 * LATENCY_RECENT_WINDOW_HOURS);
+        const priorWindowStart = OneUptimeDate.addRemoveHours(recentWindowStart, -1 * LATENCY_BASELINE_WINDOW_HOURS);
+        // Stage 1 — per-service latency profile: recent hour vs prior 24h.
+        const recentProfile = await TraceAggregationService.getServiceLatencyProfile({
+            projectId: context.projectId,
+            startTime: recentWindowStart,
+            endTime: context.now,
+            limit: LATENCY_SERVICE_LIMIT,
+        });
+        const priorProfile = await TraceAggregationService.getServiceLatencyProfile({
+            projectId: context.projectId,
+            startTime: priorWindowStart,
+            endTime: recentWindowStart,
+            limit: LATENCY_SERVICE_LIMIT,
+        });
+        /*
+         * A partial window is not "less data", it is WRONG data: a p99 and a
+         * span count computed over whatever fraction ClickHouse managed before
+         * the execution cap fired. Comparing a partial window against a
+         * complete one manufactures regressions that never happened (and hides
+         * ones that did). Skip this project's latency detection for the tick
+         * and let the next one, 15 minutes out, decide — a delayed insight is
+         * recoverable, a fabricated one (which may open a fix PR) is not.
+         */
+        if (recentProfile.isPartial || priorProfile.isPartial) {
+            logger.info(`TraceLatencyRegressionDetector: skipping project ${context.projectId.toString()} for this scan — the service latency profile query returned a possibly partial result (execution cap reached).`);
+            return [];
+        }
+        const joined = TraceLatencyRegressionDetector.joinServiceWindows(recentProfile.rows, priorProfile.rows);
+        const candidates = [];
+        for (const windows of joined) {
+            const decision = TraceLatencyRegressionDetector.evaluateRegression({
+                recentP99Ms: windows.recentP99Ms,
+                priorP99Ms: windows.priorP99Ms,
+                priorSampleCount: windows.priorSampleCount,
+            });
+            if (!decision.isRegression) {
+                continue;
+            }
+            candidates.push(await this.drillRegression({
+                context,
+                windows,
+                decision,
+                recentWindowStart,
+            }));
+        }
+        return candidates;
+    }
+    /*
+     * Drill one regressed service down to a concrete, code-shaped story:
+     * slowest CREDIBLE operation (p99 in the spike hour, sample floor
+     * applied) → the single slowest span of that operation → its whole trace
+     * → deterministic span-tree findings + code.* locations. When no
+     * operation clears the floor the insight stays at service level rather
+     * than pinning the blame — and the drill queries are skipped entirely.
+     */
+    async drillRegression(input) {
+        var _a, _b;
+        const serviceEntityId = new ObjectID(input.windows.primaryEntityId);
+        /*
+         * Candidate operations for the service — the top-list ranks by the
+         * metric (p99) and returns each operation's span count alongside it,
+         * which is exactly what the sample floor needs. Bounded: one 1-hour
+         * query pinned to the one service that regressed.
+         */
+        const topOperations = await TraceAggregationService.getAnalyticsTopList({
+            projectId: input.context.projectId,
+            startTime: input.recentWindowStart,
+            endTime: input.context.now,
+            bucketSizeInMinutes: LATENCY_RECENT_WINDOW_HOURS * 60,
+            chartType: "toplist",
+            metric: "p99Duration",
+            groupBy: ["name"],
+            serviceIds: [serviceEntityId],
+            limit: LATENCY_TOP_OPERATION_CANDIDATES,
+        });
+        const operationName = TraceLatencyRegressionDetector.pickSlowestOperation(topOperations);
+        /*
+         * Representative trace: the slowest span of the slowest operation in
+         * the spike window, then that trace's full span set. Explicit
+         * projectId in every root query — the scanner never relies on ACL.
+         */
+        let sampleTraceId = undefined;
+        let findings = [];
+        let codeLocations = [];
+        if (operationName) {
+            const slowestSpans = await SpanService.findBy({
+                query: {
+                    projectId: input.context.projectId,
+                    primaryEntityId: serviceEntityId,
+                    name: operationName,
+                    startTime: new InBetween(input.recentWindowStart, input.context.now),
+                },
+                select: {
+                    traceId: true,
+                },
+                sort: {
+                    durationUnixNano: SortOrder.Descending,
+                },
+                limit: 1,
+                skip: 0,
+                props: { isRoot: true },
+            });
+            sampleTraceId = ((_b = (_a = slowestSpans[0]) === null || _a === void 0 ? void 0 : _a.traceId) === null || _b === void 0 ? void 0 : _b.toString()) || undefined;
+        }
+        if (sampleTraceId) {
+            const traceSpans = await SpanService.findBy({
+                query: {
+                    projectId: input.context.projectId,
+                    traceId: sampleTraceId,
+                },
+                select: {
+                    spanId: true,
+                    parentSpanId: true,
+                    name: true,
+                    startTimeUnixNano: true,
+                    endTimeUnixNano: true,
+                    durationUnixNano: true,
+                    attributes: true,
+                },
+                sort: {
+                    startTimeUnixNano: SortOrder.Ascending,
+                },
+                limit: LATENCY_MAX_TRACE_SPANS,
+                skip: 0,
+                props: { isRoot: true },
+            });
+            const analyzableSpans = TraceLatencyRegressionDetector.toAnalyzableSpans(traceSpans);
+            findings = SpanTreeAnalyzer.analyzeTrace(analyzableSpans);
+            codeLocations = FixPerformanceTaskTrigger.collectCodeLocations(analyzableSpans, findings);
+        }
+        const service = await ServiceService.findOneById({
+            id: serviceEntityId,
+            select: { name: true },
+            props: { isRoot: true },
+        });
+        const serviceName = (service === null || service === void 0 ? void 0 : service.name) || null;
+        const detailLines = [
+            "**Trace latency regression detected**",
+            "",
+            `- Service: ${serviceName || input.windows.primaryEntityId}`,
+            `- Recent p99 (last ${LATENCY_RECENT_WINDOW_HOURS}h): ${input.windows.recentP99Ms.toFixed(0)} ms over ${input.windows.recentSampleCount} spans`,
+            `- Baseline p99 (prior ${LATENCY_BASELINE_WINDOW_HOURS}h): ${input.windows.priorP99Ms.toFixed(0)} ms over ${input.windows.priorSampleCount} spans`,
+            `- Regression multiplier: ${input.decision.multiplier.toFixed(1)}x`,
+        ];
+        if (operationName) {
+            detailLines.push(`- Slowest operation: \`${operationName}\``);
+        }
+        if (sampleTraceId) {
+            detailLines.push(`- Representative trace: \`${sampleTraceId}\``);
+        }
+        if (findings.length > 0) {
+            detailLines.push("", "## Span-tree findings (representative trace)", "", SpanTreeAnalyzer.renderFindingsMarkdown(findings));
+        }
+        return {
+            insightType: AIInsightType.TraceLatencyRegression,
+            fingerprint: TraceLatencyRegressionDetector.buildFingerprint(input.windows.primaryEntityId, operationName),
+            title: `Latency regression: p99 ${input.decision.multiplier.toFixed(1)}x${operationName ? ` on ${operationName}` : ""}${serviceName ? ` in ${serviceName}` : ""}`,
+            detailMarkdown: detailLines.join("\n"),
+            severity: input.decision.severity,
+            serviceName: serviceName || undefined,
+            telemetryServiceId: serviceName ? serviceEntityId : undefined,
+            traceId: sampleTraceId,
+            evidence: {
+                latency: {
+                    recentP99Ms: input.windows.recentP99Ms,
+                    baselineP99Ms: input.windows.priorP99Ms,
+                    regressionMultiplier: input.decision.multiplier,
+                    operationName,
+                    sampleTraceId,
+                    performanceFindings: findings.length > 0 ? findings : undefined,
+                    codeLocations: codeLocations.length > 0 ? codeLocations : undefined,
+                },
+            },
+        };
+    }
+}
+//# sourceMappingURL=TraceLatencyRegressionDetector.js.map

package/build/dist/Server/Utils/AI/SRE/Insights/Detectors/TraceLatencyRegressionDetector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TraceLatencyRegressionDetector.js","sourceRoot":"","sources":["../../../../../../../../Server/Utils/AI/SRE/Insights/Detectors/TraceLatencyRegressionDetector.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,kCAAkC,CAAC;AACxD,OAAO,aAAa,MAAM,8BAA8B,CAAC;AACzD,OAAO,SAAS,MAAM,gDAAgD,CAAC;AACvE,OAAO,SAAS,MAAM,gDAAgD,CAAC;AACvE,OAAO,aAAa,MAAM,0CAA0C,CAAC;AACrE,OAAO,iBAAiB,MAAM,8CAA8C,CAAC;AAO7E,OAAO,WAAW,MAAM,qCAAqC,CAAC;AAC9D,OAAO,cAAc,MAAM,wCAAwC,CAAC;AACpE,OAAO,uBAIN,MAAM,iDAAiD,CAAC;AACzD,OAAO,gBAEN,MAAM,wCAAwC,CAAC;AAChD,OAAO,yBAAyB,MAAM,iCAAiC,CAAC;AACxE,OAAO,MAAM,MAAM,oBAAoB,CAAC;AAOxC;;;;;;;;;;;;;;;;;GAiBG;AAEH,qEAAqE;AACrE,MAAM,CAAC,MAAM,2BAA2B,GAAW,CAAC,CAAC;AAErD;;;;GAIG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAW,EAAE,CAAC;AAExD;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAW,IAAI,CAAC;AAEtD,4EAA4E;AAC5E,MAAM,CAAC,MAAM,iCAAiC,GAAW,CAAC,CAAC;AAE3D,8DAA8D;AAC9D,MAAM,CAAC,MAAM,gCAAgC,GAAW,CAAC,CAAC;AAE1D;;;;GAIG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAW,GAAG,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAW,EAAE,CAAC;AAEhD;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAW,GAAG,CAAC;AAEnD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAW,EAAE,CAAC;AAE3D;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAAW,EAAE,CAAC;AAkB7D,MAAM,CAAC,OAAO,OAAO,8BAA8B;IAAnD;QACS,gBAAW,GAAkB,aAAa,CAAC,sBAAsB,CAAC;IAoY3E,CAAC;IAlYC;;;;OAIG;IACI,MAAM,CAAC,kBAAkB,CAAC,KAIhC;QACC,MAAM,UAAU,GACd,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAElE,MAAM,YAAY,GAChB,KAAK,CAAC,WAAW,IAAI,yBAAyB;YAC9C,KAAK,CAAC,gBAAgB,IAAI,8BAA8B;YACxD,KAAK,CAAC,UAAU,GAAG,CAAC;YACpB,UAAU,IAAI,iCAAiC,CAAC;QAElD,OAAO;YACL,YAAY;YACZ,UAAU;YACV,QAAQ,EACN,UAAU,IAAI,gCAAgC;gBAC5C,CAAC,CAAC,iBAAiB,CAAC,IAAI;gBACxB,CAAC,CAAC,iBAAiB,CAAC,MAAM;SAC/B,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,kBAAkB,CAC9B,UAA2C,EAC3C,SAA0C;QAE1C,MAAM,aAAa,GAA0C,IAAI,GAAG,EAAE,CAAC;QACvE,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,IAAI,GAAG,CAAC,eAAe,EAAE,CAAC;gBACxB,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAiC,EAAE,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAW,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC;YACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GACT,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC;gBACV,eAAe,EAAE,QAAQ;gBACzB,WAAW,EAAE,GAAG,CAAC,aAAa;gBAC9B,iBAAiB,EAAE,GAAG,CAAC,KAAK;gBAC5B,UAAU,EAAE,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,aAAa,KAAI,CAAC;gBACrC,gBAAgB,EAAE,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,KAAK,KAAI,CAAC;aACpC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACI,MAAM,CAAC,oBAAoB,CAChC,KAAmC;;QAEnC,MAAM,QAAQ,GAAiC,KAAK,CAAC,MAAM,CACzD,CAAC,IAA2B,EAAW,EAAE;YACvC,OAAO,CACL,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;gBACrB,IAAI,CAAC,KAAK,IAAI,kCAAkC,CACjD,CAAC;QACJ,CAAC,CACF,CAAC;QAEF,QAAQ,CAAC,IAAI,CACX,CAAC,CAAwB,EAAE,CAAwB,EAAU,EAAE;YAC7D,IAAI,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;gBACpC,OAAO,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC;YACvC,CAAC;YACD,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;gBACxB,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;YAC3B,CAAC;YACD,OAAO,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACxC,CAAC,CACF,CAAC;QAEF,OAAO,MAAA,QAAQ,CAAC,CAAC,CAAC,0CAAE,KAAK,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,iBAAiB,CAAC,KAAkB;QAChD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAU,EAAkB,EAAE;;YAC9C,MAAM,UAAU,GAA2B,EAAE,CAAC;YAE9C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;gBACjE,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBAC1C,UAAU,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,CAAA,MAAA,IAAI,CAAC,MAAM,0CAAE,QAAQ,EAAE,KAAI,EAAE;gBACrC,YAAY,EAAE,CAAA,MAAA,IAAI,CAAC,YAAY,0CAAE,QAAQ,EAAE,KAAI,SAAS;gBACxD,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,OAAS;gBACnD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,OAAS;gBAC/C,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,OAAS;gBACrD,UAAU;aACX,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,gBAAgB,CAC5B,eAAuB,EACvB,aAAiC;QAEjC,OAAO,WAAW,eAAe,IAAI,gBAAgB,CAAC,iBAAiB,CACrE,aAAa,IAAI,EAAE,CACpB,EAAE,CAAC;IACN,CAAC;IAEM,KAAK,CAAC,MAAM,CACjB,OAA2B;QAE3B,MAAM,iBAAiB,GAAS,aAAa,CAAC,cAAc,CAC1D,OAAO,CAAC,GAAG,EACX,CAAC,CAAC,GAAG,2BAA2B,CACjC,CAAC;QACF,MAAM,gBAAgB,GAAS,aAAa,CAAC,cAAc,CACzD,iBAAiB,EACjB,CAAC,CAAC,GAAG,6BAA6B,CACnC,CAAC;QAEF,mEAAmE;QACnE,MAAM,aAAa,GACjB,MAAM,uBAAuB,CAAC,wBAAwB,CAAC;YACrD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS,EAAE,iBAAiB;YAC5B,OAAO,EAAE,OAAO,CAAC,GAAG;YACpB,KAAK,EAAE,qBAAqB;SAC7B,CAAC,CAAC;QAEL,MAAM,YAAY,GAChB,MAAM,uBAAuB,CAAC,wBAAwB,CAAC;YACrD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS,EAAE,gBAAgB;YAC3B,OAAO,EAAE,iBAAiB;YAC1B,KAAK,EAAE,qBAAqB;SAC7B,CAAC,CAAC;QAEL;;;;;;;;WAQG;QACH,IAAI,aAAa,CAAC,SAAS,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;YACtD,MAAM,CAAC,IAAI,CACT,oDAAoD,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,gHAAgH,CACjM,CAAC;YACF,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,MAAM,GACV,8BAA8B,CAAC,kBAAkB,CAC/C,aAAa,CAAC,IAAI,EAClB,YAAY,CAAC,IAAI,CAClB,CAAC;QAEJ,MAAM,UAAU,GAA4B,EAAE,CAAC;QAE/C,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YAC7B,MAAM,QAAQ,GACZ,8BAA8B,CAAC,kBAAkB,CAAC;gBAChD,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;aAC3C,CAAC,CAAC;YAEL,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;gBAC3B,SAAS;YACX,CAAC;YAED,UAAU,CAAC,IAAI,CACb,MAAM,IAAI,CAAC,eAAe,CAAC;gBACzB,OAAO;gBACP,OAAO;gBACP,QAAQ;gBACR,iBAAiB;aAClB,CAAC,CACH,CAAC;QACJ,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,eAAe,CAAC,KAK7B;;QACC,MAAM,eAAe,GAAa,IAAI,QAAQ,CAC5C,KAAK,CAAC,OAAO,CAAC,eAAe,CAC9B,CAAC;QAEF;;;;;WAKG;QACH,MAAM,aAAa,GACjB,MAAM,uBAAuB,CAAC,mBAAmB,CAAC;YAChD,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS;YAClC,SAAS,EAAE,KAAK,CAAC,iBAAiB;YAClC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG;YAC1B,mBAAmB,EAAE,2BAA2B,GAAG,EAAE;YACrD,SAAS,EAAE,SAAS;YACpB,MAAM,EAAE,aAAa;YACrB,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,UAAU,EAAE,CAAC,eAAe,CAAC;YAC7B,KAAK,EAAE,gCAAgC;SACxC,CAAC,CAAC;QAEL,MAAM,aAAa,GACjB,8BAA8B,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAErE;;;;WAIG;QACH,IAAI,aAAa,GAAuB,SAAS,CAAC;QAClD,IAAI,QAAQ,GAA8B,EAAE,CAAC;QAC7C,IAAI,aAAa,GAAmC,EAAE,CAAC;QAEvD,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,YAAY,GAAgB,MAAM,WAAW,CAAC,MAAM,CAAC;gBACzD,KAAK,EAAE;oBACL,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS;oBAClC,eAAe,EAAE,eAAe;oBAChC,IAAI,EAAE,aAAa;oBACnB,SAAS,EAAE,IAAI,SAAS,CAAC,KAAK,CAAC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;iBAC5D;gBACV,MAAM,EAAE;oBACN,OAAO,EAAE,IAAI;iBACL;gBACV,IAAI,EAAE;oBACJ,gBAAgB,EAAE,SAAS,CAAC,UAAU;iBAC9B;gBACV,KAAK,EAAE,CAAC;gBACR,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;YAEH,aAAa,GAAG,CAAA,MAAA,MAAA,YAAY,CAAC,CAAC,CAAC,0CAAE,OAAO,0CAAE,QAAQ,EAAE,KAAI,SAAS,CAAC;QACpE,CAAC;QAED,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,UAAU,GAAgB,MAAM,WAAW,CAAC,MAAM,CAAC;gBACvD,KAAK,EAAE;oBACL,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS;oBAClC,OAAO,EAAE,aAAa;iBACd;gBACV,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,YAAY,EAAE,IAAI;oBAClB,IAAI,EAAE,IAAI;oBACV,iBAAiB,EAAE,IAAI;oBACvB,eAAe,EAAE,IAAI;oBACrB,gBAAgB,EAAE,IAAI;oBACtB,UAAU,EAAE,IAAI;iBACR;gBACV,IAAI,EAAE;oBACJ,iBAAiB,EAAE,SAAS,CAAC,SAAS;iBAC9B;gBACV,KAAK,EAAE,uBAAuB;gBAC9B,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;YAEH,MAAM,eAAe,GACnB,8BAA8B,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAE/D,QAAQ,GAAG,gBAAgB,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;YAC1D,aAAa,GAAG,yBAAyB,CAAC,oBAAoB,CAC5D,eAAe,EACf,QAAQ,CACT,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAmB,MAAM,cAAc,CAAC,WAAW,CAAC;YAC/D,EAAE,EAAE,eAAe;YACnB,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;YACtB,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QACH,MAAM,WAAW,GAAkB,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,KAAI,IAAI,CAAC;QAEzD,MAAM,WAAW,GAAkB;YACjC,uCAAuC;YACvC,EAAE;YACF,cAAc,WAAW,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,EAAE;YAC5D,sBAAsB,2BAA2B,OAAO,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,CAAC,iBAAiB,QAAQ;YAC/I,yBAAyB,6BAA6B,OAAO,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,OAAO,CAAC,gBAAgB,QAAQ;YAClJ,4BAA4B,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;SACpE,CAAC;QACF,IAAI,aAAa,EAAE,CAAC;YAClB,WAAW,CAAC,IAAI,CAAC,0BAA0B,aAAa,IAAI,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,aAAa,EAAE,CAAC;YAClB,WAAW,CAAC,IAAI,CAAC,6BAA6B,aAAa,IAAI,CAAC,CAAC;QACnE,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,WAAW,CAAC,IAAI,CACd,EAAE,EACF,8CAA8C,EAC9C,EAAE,EACF,gBAAgB,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAClD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,WAAW,EAAE,aAAa,CAAC,sBAAsB;YACjD,WAAW,EAAE,8BAA8B,CAAC,gBAAgB,CAC1D,KAAK,CAAC,OAAO,CAAC,eAAe,EAC7B,aAAa,CACd;YACD,KAAK,EAAE,2BAA2B,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,OAAO,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;YACjK,cAAc,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YACtC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;YACjC,WAAW,EAAE,WAAW,IAAI,SAAS;YACrC,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;YAC7D,OAAO,EAAE,aAAa;YACtB,QAAQ,EAAE;gBACR,OAAO,EAAE;oBACP,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,WAAW;oBACtC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,UAAU;oBACvC,oBAAoB,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU;oBAC/C,aAAa;oBACb,aAAa;oBACb,mBAAmB,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;oBAC/D,aAAa,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;iBACpE;aACF;SACF,CAAC;IACJ,CAAC;CACF"}

package/build/dist/Server/Utils/AI/SRE/Insights/FixRouting.js

@@ -0,0 +1,180 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import AIInsightType from "../../../../../Types/AI/AIInsightType";
+import CodeFixTaskType from "../../../../../Types/AI/CodeFixTaskType";
+import AIRunService from "../../../../Services/AIRunService";
+import TelemetryExceptionService from "../../../../Services/TelemetryExceptionService";
+import FixPerformanceTaskTrigger from "../FixPerformanceTaskTrigger";
+import FixRunBudget from "../../CodeFix/FixRunBudget";
+import logger from "../../../Logger";
+import CaptureSpan from "../../../Telemetry/CaptureSpan";
+export default class InsightFixRouting {
+    /*
+     * NEVER throws — this runs inside the scanner's per-insight routing and a
+     * failed fix enqueue must not fail (or duplicate) the scan itself. Every
+     * failure is logged and swallowed; an empty result means "no fix opened"
+     * (the insight stays ActionRequired, where the manual fix paths remain
+     * available).
+     */
+    static async routeInsightFix(data) {
+        var _a, _b, _c, _d, _e, _f, _g;
+        const { insight, project } = data;
+        try {
+            if (!insight.id || !insight.projectId) {
+                return {};
+            }
+            /*
+             * The master AI kill switch, checked BEFORE the per-feature opt-in —
+             * the same order InstrumentationTaskTrigger and InsightTriage use. A
+             * project that turned AI off gets no autonomous agent work at all,
+             * whatever a legacy per-feature flag still says. `=== false` (not
+             * `!== true`) because the column defaults to TRUE: an unselected or
+             * undefined value must never be read as "AI disabled". The scanner
+             * selects enableAi onto the project row it hands us.
+             */
+            if (project.enableAi === false) {
+                logger.debug(`AI insights: not routing a fix for insight ${insight.id.toString()} — AI is disabled for the project.`);
+                return {};
+            }
+            /*
+             * Strict opt-in — the column defaults to false, so unset/legacy rows
+             * never enqueue. Autonomous PR creation must never be default-on.
+             */
+            if (project.enableInsightFixTasks !== true) {
+                logger.debug(`AI insights: not routing a fix for insight ${insight.id.toString()} — project has not opted in to insight fix tasks.`);
+                return {};
+            }
+            const isExceptionInsight = insight.insightType === AIInsightType.NewException ||
+                insight.insightType === AIInsightType.ExceptionSpike;
+            const isLatencyInsight = insight.insightType === AIInsightType.TraceLatencyRegression;
+            /*
+             * ErrorLogSpike and MetricDrift are NEVER auto-fixed: neither carries
+             * evidence that points at a specific code path, so there is no
+             * grounded recipe to run — they route to triage + the human inbox.
+             */
+            if (!isExceptionInsight && !isLatencyInsight) {
+                return {};
+            }
+            // Exception insights are only fixable with the exception row to fix.
+            if (isExceptionInsight && !insight.telemetryExceptionId) {
+                logger.debug(`AI insights: not routing a fix for insight ${insight.id.toString()} — no telemetryExceptionId on the insight.`);
+                return {};
+            }
+            /*
+             * Latency insights are only fixable with the deterministic span-tree
+             * findings (and the trace they came from) captured at detect time.
+             */
+            const latencyFindings = ((_b = (_a = insight.evidence) === null || _a === void 0 ? void 0 : _a.latency) === null || _b === void 0 ? void 0 : _b.performanceFindings) || [];
+            const latencyCodeLocations = ((_d = (_c = insight.evidence) === null || _c === void 0 ? void 0 : _c.latency) === null || _d === void 0 ? void 0 : _d.codeLocations) || [];
+            const latencyTraceId = insight.traceId || ((_f = (_e = insight.evidence) === null || _e === void 0 ? void 0 : _e.latency) === null || _f === void 0 ? void 0 : _f.sampleTraceId);
+            if (isLatencyInsight &&
+                (latencyFindings.length === 0 || !latencyTraceId)) {
+                logger.debug(`AI insights: not routing a fix for insight ${insight.id.toString()} — the latency evidence has no span-tree findings or no trace id.`);
+                return {};
+            }
+            /*
+             * G11 guardrail: the per-project daily fix-run budget. For this
+             * AUTOMATIC trigger, over-budget is a logged skip — never an error
+             * (the creation paths below would throw the same rejection, but that
+             * lands in the catch as an unexpected failure; a budget skip is
+             * expected behavior, not a failure).
+             */
+            const budget = await FixRunBudget.getBudgetStatus(insight.projectId);
+            if (!budget.allowed) {
+                logger.debug(`AI insights: not routing a fix for insight ${insight.id.toString()} — ${FixRunBudget.describeRejection(budget)}`);
+                return {};
+            }
+            let createdRun;
+            if (isExceptionInsight) {
+                /*
+                 * Readiness precheck (provider payable, repository resolvable,
+                 * agent alive) so an unready project gets a quiet skip instead of
+                 * a thrown gate: the insight stays ActionRequired and the exception
+                 * page's manual "Fix with AI Agent" button remains available once
+                 * the project fixes its setup.
+                 */
+                const readiness = await TelemetryExceptionService.getAIFixReadiness({
+                    telemetryExceptionId: insight.telemetryExceptionId,
+                    props: { isRoot: true },
+                });
+                if (!readiness.ready) {
+                    logger.debug(`AI insights: not routing a fix for insight ${insight.id.toString()} — AI fix readiness checks failed.`);
+                    return {};
+                }
+                /*
+                 * The exception-page creation path, system-triggered (no userId in
+                 * props — the run stays system-authored): budget, readiness
+                 * re-check and per-(exception, recipe) dedupe are enforced inside.
+                 */
+                createdRun =
+                    await TelemetryExceptionService.createCodeFixRunForException({
+                        telemetryExceptionId: insight.telemetryExceptionId,
+                        props: { isRoot: true },
+                        taskType: CodeFixTaskType.FixException,
+                    });
+            }
+            else {
+                /*
+                 * The findings-based FixPerformance path: the deterministic
+                 * evidence gate already ran at detect time (the stored findings ARE
+                 * its output); repo gate, per-trace dedupe and budget are enforced
+                 * inside.
+                 */
+                createdRun =
+                    await FixPerformanceTaskTrigger.createPerformanceFixTaskFromFindings({
+                        projectId: insight.projectId,
+                        traceId: latencyTraceId,
+                        serviceName: insight.serviceName,
+                        findings: latencyFindings,
+                        codeLocations: latencyCodeLocations,
+                    });
+            }
+            if (!createdRun.id) {
+                return {};
+            }
+            /*
+             * Provenance: stamp the created run with the insight that opened it,
+             * so the run's origin is queryable from either side. Best-effort —
+             * the fix run exists either way, and the scanner links insight ->
+             * run via the returned fixAiRunId.
+             */
+            try {
+                await AIRunService.updateOneById({
+                    id: createdRun.id,
+                    data: {
+                        triggeredByAiInsightId: insight.id,
+                    },
+                    props: { isRoot: true },
+                });
+            }
+            catch (error) {
+                logger.error(`AI insights: failed to stamp fix run ${createdRun.id.toString()} with insight ${insight.id.toString()}: ${error}`);
+            }
+            logger.debug(`AI insights: queued ${isExceptionInsight ? "FixException" : "FixPerformance"} run ${createdRun.id.toString()} for insight ${insight.id.toString()}.`);
+            return { fixAiRunId: createdRun.id };
+        }
+        catch (error) {
+            /*
+             * Creation-path throws are mostly EXPECTED quiet skips for this
+             * automatic trigger (dedupe hit, readiness re-check raced, budget
+             * raced) — log at debug and leave the insight on the human path.
+             */
+            logger.debug(`AI insights: fix routing skipped for insight ${(_g = insight.id) === null || _g === void 0 ? void 0 : _g.toString()}: ${error}`);
+            return {};
+        }
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], InsightFixRouting, "routeInsightFix", null);
+//# sourceMappingURL=FixRouting.js.map

package/build/dist/Server/Utils/AI/SRE/Insights/FixRouting.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FixRouting.js","sourceRoot":"","sources":["../../../../../../../Server/Utils/AI/SRE/Insights/FixRouting.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,aAAa,MAAM,uCAAuC,CAAC;AAClE,OAAO,eAAe,MAAM,yCAAyC,CAAC;AAQtE,OAAO,YAAY,MAAM,mCAAmC,CAAC;AAC7D,OAAO,yBAEN,MAAM,gDAAgD,CAAC;AACxD,OAAO,yBAAyB,MAAM,8BAA8B,CAAC;AACrE,OAAO,YAAsC,MAAM,4BAA4B,CAAC;AAChF,OAAO,MAAM,MAAM,iBAAiB,CAAC;AACrC,OAAO,WAAW,MAAM,gCAAgC,CAAC;AAsBzD,MAAM,CAAC,OAAO,OAAO,iBAAiB;IACpC;;;;;;OAMG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,IAGnC;;QACC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;QAElC,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;gBACtC,OAAO,EAAE,CAAC;YACZ,CAAC;YAED;;;;;;;;eAQG;YACH,IAAI,OAAO,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;gBAC/B,MAAM,CAAC,KAAK,CACV,8CAA8C,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,oCAAoC,CACxG,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YAED;;;eAGG;YACH,IAAI,OAAO,CAAC,qBAAqB,KAAK,IAAI,EAAE,CAAC;gBAC3C,MAAM,CAAC,KAAK,CACV,8CAA8C,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,mDAAmD,CACvH,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,MAAM,kBAAkB,GACtB,OAAO,CAAC,WAAW,KAAK,aAAa,CAAC,YAAY;gBAClD,OAAO,CAAC,WAAW,KAAK,aAAa,CAAC,cAAc,CAAC;YAEvD,MAAM,gBAAgB,GACpB,OAAO,CAAC,WAAW,KAAK,aAAa,CAAC,sBAAsB,CAAC;YAE/D;;;;eAIG;YACH,IAAI,CAAC,kBAAkB,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC7C,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,qEAAqE;YACrE,IAAI,kBAAkB,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC;gBACxD,MAAM,CAAC,KAAK,CACV,8CAA8C,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,4CAA4C,CAChH,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YAED;;;eAGG;YACH,MAAM,eAAe,GACnB,CAAA,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,OAAO,0CAAE,mBAAmB,KAAI,EAAE,CAAC;YACvD,MAAM,oBAAoB,GACxB,CAAA,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,OAAO,0CAAE,aAAa,KAAI,EAAE,CAAC;YACjD,MAAM,cAAc,GAClB,OAAO,CAAC,OAAO,KAAI,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,OAAO,0CAAE,aAAa,CAAA,CAAC;YAE9D,IACE,gBAAgB;gBAChB,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EACjD,CAAC;gBACD,MAAM,CAAC,KAAK,CACV,8CAA8C,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,mEAAmE,CACvI,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YAED;;;;;;eAMG;YACH,MAAM,MAAM,GAAyB,MAAM,YAAY,CAAC,eAAe,CACrE,OAAO,CAAC,SAAS,CAClB,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,CAAC,KAAK,CACV,8CAA8C,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,YAAY,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAClH,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,IAAI,UAAiB,CAAC;YAEtB,IAAI,kBAAkB,EAAE,CAAC;gBACvB;;;;;;mBAMG;gBACH,MAAM,SAAS,GACb,MAAM,yBAAyB,CAAC,iBAAiB,CAAC;oBAChD,oBAAoB,EAAE,OAAO,CAAC,oBAAqB;oBACnD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;iBACxB,CAAC,CAAC;gBAEL,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;oBACrB,MAAM,CAAC,KAAK,CACV,8CAA8C,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,oCAAoC,CACxG,CAAC;oBACF,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAED;;;;mBAIG;gBACH,UAAU;oBACR,MAAM,yBAAyB,CAAC,4BAA4B,CAAC;wBAC3D,oBAAoB,EAAE,OAAO,CAAC,oBAAqB;wBACnD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;wBACvB,QAAQ,EAAE,eAAe,CAAC,YAAY;qBACvC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACN;;;;;mBAKG;gBACH,UAAU;oBACR,MAAM,yBAAyB,CAAC,oCAAoC,CAAC;wBACnE,SAAS,EAAE,OAAO,CAAC,SAAS;wBAC5B,OAAO,EAAE,cAAe;wBACxB,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,QAAQ,EAAE,eAAe;wBACzB,aAAa,EAAE,oBAAoB;qBACpC,CAAC,CAAC;YACP,CAAC;YAED,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;gBACnB,OAAO,EAAE,CAAC;YACZ,CAAC;YAED;;;;;eAKG;YACH,IAAI,CAAC;gBACH,MAAM,YAAY,CAAC,aAAa,CAAC;oBAC/B,EAAE,EAAE,UAAU,CAAC,EAAE;oBACjB,IAAI,EAAE;wBACJ,sBAAsB,EAAE,OAAO,CAAC,EAAE;qBACnC;oBACD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;iBACxB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CACV,wCAAwC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,iBAAiB,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACnH,CAAC;YACJ,CAAC;YAED,MAAM,CAAC,KAAK,CACV,uBAAuB,kBAAkB,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,gBAAgB,QAAQ,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CACtJ,CAAC;YAEF,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf;;;;eAIG;YACH,MAAM,CAAC,KAAK,CACV,gDAAgD,MAAA,OAAO,CAAC,EAAE,0CAAE,QAAQ,EAAE,KAAK,KAAK,EAAE,CACnF,CAAC;YACF,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAjMqB;IADnB,WAAW,EAAE;;;;8CAiMb"}