@oneuptime/common 11.7.4 → 12.0.0

package/build/dist/Server/Services/AutoRemediationRuleEngineService.js

@@ -0,0 +1,917 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import Alert from "../../Models/DatabaseModels/Alert";
+import AutoRemediationSuggestion from "../../Models/DatabaseModels/AutoRemediationSuggestion";
+import Incident from "../../Models/DatabaseModels/Incident";
+import { AlertFeedEventType } from "../../Models/DatabaseModels/AlertFeed";
+import { IncidentFeedEventType } from "../../Models/DatabaseModels/IncidentFeed";
+import AIRunType from "../../Types/AI/AIRunType";
+import AutoRemediationExecutionMode from "../../Types/AutoRemediation/AutoRemediationExecutionMode";
+import AutoRemediationSuggestionStatus from "../../Types/AutoRemediation/AutoRemediationSuggestionStatus";
+import AutoRemediationSuggestionType from "../../Types/AutoRemediation/AutoRemediationSuggestionType";
+import AutoRemediationVerificationStatus from "../../Types/AutoRemediation/AutoRemediationVerificationStatus";
+import AutoRemediationTriggerEntity from "../../Types/AutoRemediation/AutoRemediationTriggerEntity";
+import { Indigo500 } from "../../Types/BrandColors";
+import OneUptimeDate from "../../Types/Date";
+import QueryHelper from "../Types/Database/QueryHelper";
+import AlertService from "./AlertService";
+import IncidentService from "./IncidentService";
+import AlertFeedService from "./AlertFeedService";
+import AutoRemediationRuleService from "./AutoRemediationRuleService";
+import AutoRemediationSuggestionService from "./AutoRemediationSuggestionService";
+import IncidentFeedService from "./IncidentFeedService";
+import LlmProviderService from "./LlmProviderService";
+import MonitorService from "./MonitorService";
+import ProjectService from "./ProjectService";
+import RunbookRuleEngineService from "./RunbookRuleEngineService";
+import AIInvestigationQueue from "../Utils/AI/SRE/InvestigationQueue";
+import Semaphore from "../Infrastructure/Semaphore";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import logger from "../Utils/Logger";
+/*
+ * Guardrails (minimal G1 for auto-remediation Phase 1):
+ * - At most this many suggestions per incident/alert, ever. Stops rule
+ *   fan-out and repeated matching from flooding a subject with proposals.
+ */
+export const MAX_SUGGESTIONS_PER_SUBJECT = 3;
+/*
+ * - A FullAuto rule that has auto-executed this many times in the last hour
+ *   is downgraded to Suggest for the next match (remediation-storm circuit
+ *   breaker). A human gets pulled back into the loop instead of a runbook
+ *   being started in a tight flap loop.
+ */
+export const MAX_AUTO_EXECUTIONS_PER_RULE_PER_HOUR = 3;
+/*
+ * How long the subject's monitors get to recover after a remediation
+ * runbook starts, when the rule does not set its own window. The verifier
+ * fails the remediation past this deadline — escalation continues as
+ * normal either way.
+ */
+export const DEFAULT_VERIFICATION_WINDOW_MINUTES = 15;
+class AutoRemediationRuleEngineServiceClass {
+    /*
+     * Reload an incident and run the rules against it. Used by the
+     * post-investigation remediation hand-off (RemediationHandoff), which
+     * holds only the subject id: the create-hook's in-memory model is long
+     * gone by the time the investigation settles, and the reload also picks
+     * up edits made while the investigation ran. The select mirrors exactly
+     * what doesIncidentMatchRule reads — keep the two in sync.
+     */
+    async applyRulesToIncidentById(data) {
+        const incident = await IncidentService.findOneById({
+            id: data.incidentId,
+            select: {
+                _id: true,
+                projectId: true,
+                title: true,
+                description: true,
+                incidentSeverityId: true,
+                monitors: { _id: true },
+                labels: { _id: true },
+            },
+            props: { isRoot: true },
+        });
+        if (!incident) {
+            return;
+        }
+        await this.applyRulesToIncident(incident);
+    }
+    // The alert twin of applyRulesToIncidentById — mirrors doesAlertMatchRule.
+    async applyRulesToAlertById(data) {
+        const alert = await AlertService.findOneById({
+            id: data.alertId,
+            select: {
+                _id: true,
+                projectId: true,
+                title: true,
+                description: true,
+                alertSeverityId: true,
+                monitorId: true,
+                labels: { _id: true },
+            },
+            props: { isRoot: true },
+        });
+        if (!alert) {
+            return;
+        }
+        await this.applyRulesToAlert(alert);
+    }
+    async applyRulesToIncident(incident) {
+        var _a, _b;
+        if (!incident.id || !incident.projectId) {
+            return;
+        }
+        try {
+            await this.applyRules({
+                projectId: incident.projectId,
+                triggerEntityType: AutoRemediationTriggerEntity.Incident,
+                incident,
+            });
+        }
+        catch (error) {
+            logger.error(`Error applying auto-remediation rules: ${error}`, {
+                projectId: (_a = incident.projectId) === null || _a === void 0 ? void 0 : _a.toString(),
+                incidentId: (_b = incident.id) === null || _b === void 0 ? void 0 : _b.toString(),
+            });
+        }
+    }
+    async applyRulesToAlert(alert) {
+        var _a, _b;
+        if (!alert.id || !alert.projectId) {
+            return;
+        }
+        try {
+            await this.applyRules({
+                projectId: alert.projectId,
+                triggerEntityType: AutoRemediationTriggerEntity.Alert,
+                alert,
+            });
+        }
+        catch (error) {
+            logger.error(`Error applying auto-remediation rules: ${error}`, {
+                projectId: (_a = alert.projectId) === null || _a === void 0 ? void 0 : _a.toString(),
+                alertId: (_b = alert.id) === null || _b === void 0 ? void 0 : _b.toString(),
+            });
+        }
+    }
+    async applyRules(data) {
+        var _a, _b, _c, _d, _e, _f, _g, _h;
+        /*
+         * Project-level kill switch. === false so a missing column (older rows,
+         * self-hosted defaults) counts as enabled — same semantics as enableAi.
+         */
+        const project = await ProjectService.findOneById({
+            id: data.projectId,
+            select: {
+                enableAi: true,
+                enableAutoRemediation: true,
+                enableAiCommandExecution: true,
+            },
+            props: { isRoot: true },
+        });
+        if (!project || project.enableAutoRemediation === false) {
+            return;
+        }
+        const rules = await AutoRemediationRuleService.findBy({
+            query: {
+                projectId: data.projectId,
+                isEnabled: true,
+                triggerEntityType: data.triggerEntityType,
+            },
+            props: { isRoot: true },
+            select: {
+                _id: true,
+                name: true,
+                executionMode: true,
+                aiSelectsRunbook: true,
+                aiComposesCommands: true,
+                verificationWindowMinutes: true,
+                autoResolveOnVerifiedRecovery: true,
+                titlePattern: true,
+                descriptionPattern: true,
+                monitors: { _id: true },
+                incidentSeverities: { _id: true },
+                alertSeverities: { _id: true },
+                labels: { _id: true },
+                monitorLabels: { _id: true },
+                runbooks: { _id: true, name: true },
+            },
+            limit: 100,
+            skip: 0,
+        });
+        if (rules.length === 0) {
+            return;
+        }
+        const matchedRules = [];
+        for (const rule of rules) {
+            const isMatch = data.incident
+                ? await this.doesIncidentMatchRule(data.incident, rule)
+                : data.alert
+                    ? await this.doesAlertMatchRule(data.alert, rule)
+                    : false;
+            if (isMatch) {
+                matchedRules.push(rule);
+            }
+        }
+        if (matchedRules.length === 0) {
+            return;
+        }
+        const linkage = {
+            incidentId: ((_a = data.incident) === null || _a === void 0 ? void 0 : _a.id) || undefined,
+            alertId: ((_b = data.alert) === null || _b === void 0 ? void 0 : _b.id) || undefined,
+        };
+        /*
+         * Per-subject cap + per-rule dedupe. A rule that already produced a
+         * suggestion on this subject never produces another (a dismissal is a
+         * human "no" — do not re-ask).
+         */
+        const existingSuggestions = await AutoRemediationSuggestionService.findBy({
+            query: linkage.incidentId
+                ? { incidentId: linkage.incidentId }
+                : { alertId: linkage.alertId },
+            props: { isRoot: true },
+            select: {
+                _id: true,
+                autoRemediationRuleId: true,
+            },
+            limit: MAX_SUGGESTIONS_PER_SUBJECT * 10,
+            skip: 0,
+        });
+        let remainingBudget = MAX_SUGGESTIONS_PER_SUBJECT - existingSuggestions.length;
+        if (remainingBudget <= 0) {
+            logger.debug(`AutoRemediationRuleEngine: suggestion cap reached for subject; skipping ${matchedRules.length} matched rule(s).`, {
+                projectId: data.projectId.toString(),
+                incidentId: (_c = linkage.incidentId) === null || _c === void 0 ? void 0 : _c.toString(),
+                alertId: (_d = linkage.alertId) === null || _d === void 0 ? void 0 : _d.toString(),
+            });
+            return;
+        }
+        const ruleIdsWithExistingSuggestion = new Set(existingSuggestions
+            .map((s) => {
+            var _a;
+            return ((_a = s.autoRemediationRuleId) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+        })
+            .filter((id) => {
+            return id !== "";
+        }));
+        // Lazily evaluated once: AI rules need AI enabled and a provider.
+        let aiAvailable = null;
+        for (const rule of matchedRules) {
+            if (remainingBudget <= 0) {
+                break;
+            }
+            if (rule.id && ruleIdsWithExistingSuggestion.has(rule.id.toString())) {
+                continue;
+            }
+            /*
+             * Command composition wins over runbook selection when a rule has
+             * both flags — one run per rule, never two.
+             */
+            if (rule.aiComposesCommands) {
+                if (aiAvailable === null) {
+                    aiAvailable =
+                        project.enableAi !== false &&
+                            (await LlmProviderService.getLLMProviderForProject(data.projectId)) !== null;
+                }
+                if (!aiAvailable) {
+                    logger.debug(`AutoRemediationRuleEngine: skipping AI command rule ${(_e = rule.id) === null || _e === void 0 ? void 0 : _e.toString()} — AI disabled or no LLM provider configured.`, { projectId: data.projectId.toString() });
+                    continue;
+                }
+                /*
+                 * Opt-in semantics (=== true): the project must have explicitly
+                 * enabled AI command execution, on top of the AI/auto-remediation
+                 * kill switches.
+                 */
+                if (project.enableAiCommandExecution !== true) {
+                    logger.debug(`AutoRemediationRuleEngine: skipping AI command rule ${(_f = rule.id) === null || _f === void 0 ? void 0 : _f.toString()} — the project has not enabled AI command execution.`, { projectId: data.projectId.toString() });
+                    continue;
+                }
+                await this.startAiCommandRun({
+                    projectId: data.projectId,
+                    rule,
+                    linkage,
+                });
+                remainingBudget -= 1;
+                continue;
+            }
+            if (rule.aiSelectsRunbook) {
+                if (aiAvailable === null) {
+                    aiAvailable =
+                        project.enableAi !== false &&
+                            (await LlmProviderService.getLLMProviderForProject(data.projectId)) !== null;
+                }
+                if (!aiAvailable) {
+                    logger.debug(`AutoRemediationRuleEngine: skipping AI rule ${(_g = rule.id) === null || _g === void 0 ? void 0 : _g.toString()} — AI disabled or no LLM provider configured.`, { projectId: data.projectId.toString() });
+                    continue;
+                }
+                await this.startAiPlanning({
+                    projectId: data.projectId,
+                    rule,
+                    linkage,
+                });
+                remainingBudget -= 1;
+                continue;
+            }
+            // Deterministic rule: propose or start every attached runbook.
+            const runbooks = rule.runbooks || [];
+            if (runbooks.length === 0) {
+                logger.warn(`AutoRemediationRuleEngine: rule ${(_h = rule.id) === null || _h === void 0 ? void 0 : _h.toString()} matched but has no runbooks attached and does not use AI selection; skipping.`, { projectId: data.projectId.toString() });
+                continue;
+            }
+            if (rule.executionMode === AutoRemediationExecutionMode.FullAuto) {
+                remainingBudget -= await this.applyFullAutoRule({
+                    projectId: data.projectId,
+                    rule,
+                    runbooks,
+                    linkage,
+                    budget: remainingBudget,
+                });
+                continue;
+            }
+            for (const runbook of runbooks) {
+                if (remainingBudget <= 0) {
+                    break;
+                }
+                if (!runbook.id) {
+                    continue;
+                }
+                await this.suggestRunbook({
+                    projectId: data.projectId,
+                    rule,
+                    runbook,
+                    linkage,
+                    downgradedByCircuitBreaker: false,
+                });
+                remainingBudget -= 1;
+            }
+        }
+    }
+    /*
+     * FullAuto with the hourly circuit breaker. The whole
+     * count-decide-execute-record section is serialized per rule through a
+     * distributed mutex: concurrent incident creations (flap storm, multiple
+     * App pods) would otherwise all read a stale count and blow past the cap
+     * together. The AutoExecuted suggestion rows are written inside the lock,
+     * so the next holder's count sees them. If the lock cannot be acquired
+     * (e.g. Redis unavailable) the engine degrades to the unserialized check
+     * rather than dropping remediation entirely — the breaker is
+     * defense-in-depth, not a correctness invariant.
+     *
+     * Returns how many suggestions were consumed from the per-subject budget.
+     */
+    async applyFullAutoRule(data) {
+        var _a, _b, _c;
+        let mutex = null;
+        try {
+            mutex = await Semaphore.lock({
+                key: `auto-remediation-rule-${(_a = data.rule.id) === null || _a === void 0 ? void 0 : _a.toString()}`,
+                namespace: "AutoRemediationRuleEngine",
+                lockTimeout: 60 * 1000,
+                acquireTimeout: 20 * 1000,
+            });
+        }
+        catch (error) {
+            logger.warn(`AutoRemediationRuleEngine: could not acquire circuit-breaker lock for rule ${(_b = data.rule.id) === null || _b === void 0 ? void 0 : _b.toString()} — continuing without cross-pod serialization: ${error}`, { projectId: data.projectId.toString() });
+        }
+        let consumed = 0;
+        try {
+            /*
+             * Count inside the lock, and keep counting in-memory as this
+             * invocation executes — a rule with several attached runbooks must
+             * not execute all of them off one stale pre-loop count.
+             */
+            let executedInWindow = (await AutoRemediationSuggestionService.countBy({
+                query: {
+                    autoRemediationRuleId: data.rule.id,
+                    status: AutoRemediationSuggestionStatus.AutoExecuted,
+                    createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeHoursAgo(1)),
+                },
+                props: { isRoot: true },
+            })).toNumber();
+            for (const runbook of data.runbooks) {
+                if (consumed >= data.budget) {
+                    break;
+                }
+                if (!runbook.id) {
+                    continue;
+                }
+                if (executedInWindow >= MAX_AUTO_EXECUTIONS_PER_RULE_PER_HOUR) {
+                    logger.warn(`AutoRemediationRuleEngine: circuit breaker tripped for rule ${(_c = data.rule.id) === null || _c === void 0 ? void 0 : _c.toString()} (${executedInWindow} auto-executions in the last hour); downgrading to Suggest.`, { projectId: data.projectId.toString() });
+                    await this.suggestRunbook({
+                        projectId: data.projectId,
+                        rule: data.rule,
+                        runbook,
+                        linkage: data.linkage,
+                        downgradedByCircuitBreaker: true,
+                    });
+                }
+                else {
+                    await this.autoExecuteRunbook({
+                        projectId: data.projectId,
+                        rule: data.rule,
+                        runbook,
+                        linkage: data.linkage,
+                    });
+                    /*
+                     * Counted even when the start failed (no row written): within
+                     * this invocation the fail direction of a breaker is to close
+                     * early, never to keep firing.
+                     */
+                    executedInWindow += 1;
+                }
+                consumed += 1;
+            }
+        }
+        finally {
+            if (mutex) {
+                try {
+                    await Semaphore.release(mutex);
+                }
+                catch (error) {
+                    logger.error(`AutoRemediationRuleEngine: failed to release circuit-breaker lock: ${error}`, { projectId: data.projectId.toString() });
+                }
+            }
+        }
+        return consumed;
+    }
+    /*
+     * AI path: create a Planning suggestion and enqueue a RemediationPlan run
+     * on the durable AI queue. The planning runner fills in the picked runbook
+     * and rationale (or NoneApplicable). AI-picked runbooks are always
+     * suggest-only — never full-auto.
+     */
+    async startAiPlanning(data) {
+        const suggestion = new AutoRemediationSuggestion();
+        suggestion.projectId = data.projectId;
+        suggestion.autoRemediationRuleId = data.rule.id;
+        suggestion.ruleNameSnapshot = data.rule.name || "Auto Remediation Rule";
+        suggestion.status = AutoRemediationSuggestionStatus.Planning;
+        suggestion.executionMode = AutoRemediationExecutionMode.Suggest;
+        suggestion.verificationWindowMinutes =
+            data.rule.verificationWindowMinutes ||
+                DEFAULT_VERIFICATION_WINDOW_MINUTES;
+        suggestion.autoResolveOnRecovery =
+            data.rule.autoResolveOnVerifiedRecovery === true;
+        if (data.linkage.incidentId) {
+            suggestion.incidentId = data.linkage.incidentId;
+        }
+        if (data.linkage.alertId) {
+            suggestion.alertId = data.linkage.alertId;
+        }
+        const created = await AutoRemediationSuggestionService.create({
+            data: suggestion,
+            props: { isRoot: true },
+        });
+        const aiRunId = await AIInvestigationQueue.enqueue({
+            projectId: data.projectId,
+            subjectIncidentId: data.linkage.incidentId,
+            subjectAlertId: data.linkage.alertId,
+            subjectAutoRemediationSuggestionId: created.id,
+        });
+        if (!aiRunId) {
+            /*
+             * Budget exhausted or run creation failed — do not leave the
+             * suggestion stuck in Planning forever.
+             */
+            await AutoRemediationSuggestionService.updateOneById({
+                id: created.id,
+                data: {
+                    status: AutoRemediationSuggestionStatus.NoneApplicable,
+                    rationaleMarkdown: "AI planning could not be started — the daily autonomous AI budget is exhausted or no run could be queued. Re-enable by raising the budget or waiting for the daily reset.",
+                },
+                props: { isRoot: true },
+            });
+            return;
+        }
+        await AutoRemediationSuggestionService.updateOneById({
+            id: created.id,
+            data: {
+                aiRunId,
+            },
+            props: { isRoot: true },
+        });
+        await this.postFeedItem({
+            projectId: data.projectId,
+            linkage: data.linkage,
+            markdown: `⚡ **Auto Remediation Rule "${data.rule.name}" matched.** AI is picking the most applicable runbook — a suggestion will appear here shortly.`,
+            pingWorkspace: false,
+        });
+    }
+    /*
+     * AI command path (rules with aiComposesCommands): create a Planning
+     * CommandPlan suggestion and enqueue a RemediationExecution run. The
+     * execution runner decides Suggest vs FullAuto from the rule's execution
+     * mode + allowlist + circuit breaker at run time; the suggestion's
+     * executionMode snapshots the rule's intent for display.
+     */
+    async startAiCommandRun(data) {
+        const suggestion = new AutoRemediationSuggestion();
+        suggestion.projectId = data.projectId;
+        suggestion.autoRemediationRuleId = data.rule.id;
+        suggestion.ruleNameSnapshot = data.rule.name || "Auto Remediation Rule";
+        suggestion.status = AutoRemediationSuggestionStatus.Planning;
+        suggestion.suggestionType = AutoRemediationSuggestionType.CommandPlan;
+        suggestion.executionMode =
+            data.rule.executionMode === AutoRemediationExecutionMode.FullAuto
+                ? AutoRemediationExecutionMode.FullAuto
+                : AutoRemediationExecutionMode.Suggest;
+        suggestion.verificationWindowMinutes =
+            data.rule.verificationWindowMinutes ||
+                DEFAULT_VERIFICATION_WINDOW_MINUTES;
+        suggestion.autoResolveOnRecovery =
+            data.rule.autoResolveOnVerifiedRecovery === true;
+        if (data.linkage.incidentId) {
+            suggestion.incidentId = data.linkage.incidentId;
+        }
+        if (data.linkage.alertId) {
+            suggestion.alertId = data.linkage.alertId;
+        }
+        const created = await AutoRemediationSuggestionService.create({
+            data: suggestion,
+            props: { isRoot: true },
+        });
+        const aiRunId = await AIInvestigationQueue.enqueue({
+            projectId: data.projectId,
+            subjectIncidentId: data.linkage.incidentId,
+            subjectAlertId: data.linkage.alertId,
+            subjectAutoRemediationSuggestionId: created.id,
+            remediationRunType: AIRunType.RemediationExecution,
+        });
+        if (!aiRunId) {
+            await AutoRemediationSuggestionService.updateOneById({
+                id: created.id,
+                data: {
+                    status: AutoRemediationSuggestionStatus.NoneApplicable,
+                    rationaleMarkdown: "The AI command run could not be started — the daily autonomous AI budget is exhausted or no run could be queued. Re-enable by raising the budget or waiting for the daily reset.",
+                },
+                props: { isRoot: true },
+            });
+            return;
+        }
+        await AutoRemediationSuggestionService.updateOneById({
+            id: created.id,
+            data: {
+                aiRunId,
+            },
+            props: { isRoot: true },
+        });
+        await this.postFeedItem({
+            projectId: data.projectId,
+            linkage: data.linkage,
+            markdown: `⚡ **Auto Remediation Rule "${data.rule.name}" matched.** AI is diagnosing the signal and composing remediation commands — a suggestion will appear here shortly.`,
+            pingWorkspace: false,
+        });
+    }
+    // Deterministic Suggest path: propose the runbook for one-click approval.
+    async suggestRunbook(data) {
+        const suggestion = new AutoRemediationSuggestion();
+        suggestion.projectId = data.projectId;
+        suggestion.autoRemediationRuleId = data.rule.id;
+        suggestion.ruleNameSnapshot = data.rule.name || "Auto Remediation Rule";
+        suggestion.runbookId = data.runbook.id;
+        suggestion.runbookNameSnapshot = data.runbook.name || "Runbook";
+        suggestion.status = AutoRemediationSuggestionStatus.Suggested;
+        suggestion.executionMode = AutoRemediationExecutionMode.Suggest;
+        suggestion.verificationWindowMinutes =
+            data.rule.verificationWindowMinutes ||
+                DEFAULT_VERIFICATION_WINDOW_MINUTES;
+        suggestion.autoResolveOnRecovery =
+            data.rule.autoResolveOnVerifiedRecovery === true;
+        suggestion.rationaleMarkdown = data.downgradedByCircuitBreaker
+            ? "Proposed by a FullAuto rule that was downgraded to suggest-only by the hourly circuit breaker (too many recent auto-executions). Approve to start the runbook."
+            : "Proposed by a matching auto-remediation rule. Approve to start the runbook.";
+        if (data.linkage.incidentId) {
+            suggestion.incidentId = data.linkage.incidentId;
+        }
+        if (data.linkage.alertId) {
+            suggestion.alertId = data.linkage.alertId;
+        }
+        await AutoRemediationSuggestionService.create({
+            data: suggestion,
+            props: { isRoot: true },
+        });
+        await this.postFeedItem({
+            projectId: data.projectId,
+            linkage: data.linkage,
+            markdown: `⚡ **Auto Remediation Rule "${data.rule.name}" proposed runbook "${data.runbook.name}".** Review and approve it with one click to start remediation.`,
+            pingWorkspace: true,
+        });
+    }
+    /*
+     * FullAuto path: start the runbook immediately. startRunbookFor re-checks
+     * project ownership, isEnabled and non-empty steps, so a stale or
+     * cross-project runbook reference can never execute.
+     */
+    async autoExecuteRunbook(data) {
+        var _a, _b;
+        const runbookLinkage = {};
+        if (data.linkage.incidentId) {
+            runbookLinkage.incidentId = data.linkage.incidentId;
+        }
+        if (data.linkage.alertId) {
+            runbookLinkage.alertId = data.linkage.alertId;
+        }
+        const execution = await RunbookRuleEngineService.startRunbookFor({
+            projectId: data.projectId,
+            runbookId: data.runbook.id,
+            linkage: runbookLinkage,
+        });
+        if (!execution) {
+            logger.warn(`AutoRemediationRuleEngine: FullAuto rule ${(_a = data.rule.id) === null || _a === void 0 ? void 0 : _a.toString()} could not start runbook ${(_b = data.runbook.id) === null || _b === void 0 ? void 0 : _b.toString()} (missing, disabled, cross-project or empty).`, { projectId: data.projectId.toString() });
+            await this.postFeedItem({
+                projectId: data.projectId,
+                linkage: data.linkage,
+                markdown: `⚡ **Auto Remediation Rule "${data.rule.name}" matched, but runbook "${data.runbook.name}" could not be started** (it may be disabled or have no steps).`,
+                pingWorkspace: false,
+            });
+            return;
+        }
+        const suggestion = new AutoRemediationSuggestion();
+        suggestion.projectId = data.projectId;
+        suggestion.autoRemediationRuleId = data.rule.id;
+        suggestion.ruleNameSnapshot = data.rule.name || "Auto Remediation Rule";
+        suggestion.runbookId = data.runbook.id;
+        suggestion.runbookNameSnapshot = data.runbook.name || "Runbook";
+        suggestion.status = AutoRemediationSuggestionStatus.AutoExecuted;
+        suggestion.executionMode = AutoRemediationExecutionMode.FullAuto;
+        suggestion.runbookExecutionId = execution.id;
+        suggestion.verificationWindowMinutes =
+            data.rule.verificationWindowMinutes ||
+                DEFAULT_VERIFICATION_WINDOW_MINUTES;
+        suggestion.autoResolveOnRecovery =
+            data.rule.autoResolveOnVerifiedRecovery === true;
+        // The runbook is already running — verification starts now.
+        suggestion.verificationStatus = AutoRemediationVerificationStatus.Pending;
+        suggestion.verificationDeadlineAt = OneUptimeDate.addRemoveMinutes(OneUptimeDate.getCurrentDate(), data.rule.verificationWindowMinutes ||
+            DEFAULT_VERIFICATION_WINDOW_MINUTES);
+        suggestion.rationaleMarkdown =
+            "Automatically executed by a FullAuto auto-remediation rule.";
+        if (data.linkage.incidentId) {
+            suggestion.incidentId = data.linkage.incidentId;
+        }
+        if (data.linkage.alertId) {
+            suggestion.alertId = data.linkage.alertId;
+        }
+        await AutoRemediationSuggestionService.create({
+            data: suggestion,
+            props: { isRoot: true },
+        });
+        await this.postFeedItem({
+            projectId: data.projectId,
+            linkage: data.linkage,
+            markdown: `⚡ **Auto Remediation Rule "${data.rule.name}" automatically started runbook "${data.runbook.name}".** Follow its progress on the runbook execution page.`,
+            pingWorkspace: true,
+        });
+    }
+    // Feed posting is best-effort observability — the feed services never throw.
+    async postFeedItem(data) {
+        var _a, _b;
+        try {
+            if (data.linkage.incidentId) {
+                await IncidentFeedService.createIncidentFeedItem({
+                    incidentId: data.linkage.incidentId,
+                    projectId: data.projectId,
+                    incidentFeedEventType: IncidentFeedEventType.AutoRemediation,
+                    displayColor: Indigo500,
+                    feedInfoInMarkdown: data.markdown,
+                    workspaceNotification: {
+                        sendWorkspaceNotification: data.pingWorkspace,
+                    },
+                });
+            }
+            else if (data.linkage.alertId) {
+                await AlertFeedService.createAlertFeedItem({
+                    alertId: data.linkage.alertId,
+                    projectId: data.projectId,
+                    alertFeedEventType: AlertFeedEventType.AutoRemediation,
+                    displayColor: Indigo500,
+                    feedInfoInMarkdown: data.markdown,
+                    workspaceNotification: {
+                        sendWorkspaceNotification: data.pingWorkspace,
+                    },
+                });
+            }
+        }
+        catch (error) {
+            logger.error(`AutoRemediationRuleEngine: failed to create feed item: ${error}`, {
+                projectId: data.projectId.toString(),
+                incidentId: (_a = data.linkage.incidentId) === null || _a === void 0 ? void 0 : _a.toString(),
+                alertId: (_b = data.linkage.alertId) === null || _b === void 0 ? void 0 : _b.toString(),
+            });
+        }
+    }
+    /*
+     * Matching. Every criterion is skip-if-empty with AND semantics across
+     * criteria — the same shape as IncidentOnCallRuleEngineService.
+     */
+    async doesIncidentMatchRule(incident, rule) {
+        // Monitors: incident must come from at least one of the rule's monitors.
+        if (rule.monitors && rule.monitors.length > 0) {
+            if (!incident.monitors || incident.monitors.length === 0) {
+                return false;
+            }
+            const ruleMonitorIds = rule.monitors.map((m) => {
+                var _a;
+                return ((_a = m.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+            });
+            const incidentMonitorIds = incident.monitors.map((m) => {
+                var _a;
+                return ((_a = m.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+            });
+            const hasMatch = ruleMonitorIds.some((id) => {
+                return incidentMonitorIds.includes(id);
+            });
+            if (!hasMatch) {
+                return false;
+            }
+        }
+        // Severity
+        if (rule.incidentSeverities && rule.incidentSeverities.length > 0) {
+            if (!incident.incidentSeverityId) {
+                return false;
+            }
+            const severityIds = rule.incidentSeverities.map((s) => {
+                var _a;
+                return ((_a = s.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+            });
+            if (!severityIds.includes(incident.incidentSeverityId.toString())) {
+                return false;
+            }
+        }
+        // Entity labels
+        if (rule.labels && rule.labels.length > 0) {
+            if (!incident.labels || incident.labels.length === 0) {
+                return false;
+            }
+            const ruleLabelIds = rule.labels.map((l) => {
+                var _a;
+                return ((_a = l.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+            });
+            const incidentLabelIds = incident.labels.map((l) => {
+                var _a;
+                return ((_a = l.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+            });
+            const hasMatch = ruleLabelIds.some((id) => {
+                return incidentLabelIds.includes(id);
+            });
+            if (!hasMatch) {
+                return false;
+            }
+        }
+        // Monitor labels: any of the incident's monitors carrying one is enough.
+        if (rule.monitorLabels && rule.monitorLabels.length > 0) {
+            if (!incident.monitors || incident.monitors.length === 0) {
+                return false;
+            }
+            let anyMonitorMatches = false;
+            for (const incidentMonitor of incident.monitors) {
+                if (!incidentMonitor.id) {
+                    continue;
+                }
+                if (await this.doesMonitorCarryAnyLabel(incidentMonitor.id, rule.monitorLabels)) {
+                    anyMonitorMatches = true;
+                    break;
+                }
+            }
+            if (!anyMonitorMatches) {
+                return false;
+            }
+        }
+        if (rule.titlePattern) {
+            if (!incident.title ||
+                !this.testRegex(rule.titlePattern, incident.title, rule)) {
+                return false;
+            }
+        }
+        if (rule.descriptionPattern) {
+            if (!incident.description ||
+                !this.testRegex(rule.descriptionPattern, incident.description, rule)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    async doesAlertMatchRule(alert, rule) {
+        // Monitors: alerts carry a single scalar monitorId.
+        if (rule.monitors && rule.monitors.length > 0) {
+            if (!alert.monitorId) {
+                return false;
+            }
+            const monitorIds = rule.monitors.map((m) => {
+                var _a;
+                return ((_a = m.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+            });
+            if (!monitorIds.includes(alert.monitorId.toString())) {
+                return false;
+            }
+        }
+        // Severity
+        if (rule.alertSeverities && rule.alertSeverities.length > 0) {
+            if (!alert.alertSeverityId) {
+                return false;
+            }
+            const severityIds = rule.alertSeverities.map((s) => {
+                var _a;
+                return ((_a = s.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+            });
+            if (!severityIds.includes(alert.alertSeverityId.toString())) {
+                return false;
+            }
+        }
+        // Entity labels
+        if (rule.labels && rule.labels.length > 0) {
+            if (!alert.labels || alert.labels.length === 0) {
+                return false;
+            }
+            const ruleLabelIds = rule.labels.map((l) => {
+                var _a;
+                return ((_a = l.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+            });
+            const alertLabelIds = alert.labels.map((l) => {
+                var _a;
+                return ((_a = l.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+            });
+            const hasMatch = ruleLabelIds.some((id) => {
+                return alertLabelIds.includes(id);
+            });
+            if (!hasMatch) {
+                return false;
+            }
+        }
+        // Monitor labels
+        if (rule.monitorLabels && rule.monitorLabels.length > 0) {
+            if (!alert.monitorId) {
+                return false;
+            }
+            if (!(await this.doesMonitorCarryAnyLabel(alert.monitorId, rule.monitorLabels))) {
+                return false;
+            }
+        }
+        if (rule.titlePattern) {
+            if (!alert.title ||
+                !this.testRegex(rule.titlePattern, alert.title, rule)) {
+                return false;
+            }
+        }
+        if (rule.descriptionPattern) {
+            if (!alert.description ||
+                !this.testRegex(rule.descriptionPattern, alert.description, rule)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    async doesMonitorCarryAnyLabel(monitorId, ruleMonitorLabels) {
+        const monitor = await MonitorService.findOneById({
+            id: monitorId,
+            select: {
+                labels: { _id: true },
+            },
+            props: { isRoot: true },
+        });
+        if (!monitor || !monitor.labels || monitor.labels.length === 0) {
+            return false;
+        }
+        const ruleLabelIds = ruleMonitorLabels.map((l) => {
+            var _a;
+            return ((_a = l.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+        });
+        const monitorLabelIds = monitor.labels.map((l) => {
+            var _a;
+            return ((_a = l.id) === null || _a === void 0 ? void 0 : _a.toString()) || "";
+        });
+        return ruleLabelIds.some((id) => {
+            return monitorLabelIds.includes(id);
+        });
+    }
+    testRegex(pattern, value, rule) {
+        try {
+            const regex = new RegExp(pattern, "i");
+            return regex.test(value);
+        }
+        catch (_a) {
+            logger.warn(`Invalid regex pattern in auto-remediation rule ${rule.id}: ${pattern}`);
+            return false;
+        }
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], AutoRemediationRuleEngineServiceClass.prototype, "applyRulesToIncidentById", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], AutoRemediationRuleEngineServiceClass.prototype, "applyRulesToAlertById", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Incident]),
+    __metadata("design:returntype", Promise)
+], AutoRemediationRuleEngineServiceClass.prototype, "applyRulesToIncident", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Alert]),
+    __metadata("design:returntype", Promise)
+], AutoRemediationRuleEngineServiceClass.prototype, "applyRulesToAlert", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], AutoRemediationRuleEngineServiceClass.prototype, "applyRules", null);
+const AutoRemediationRuleEngineService = new AutoRemediationRuleEngineServiceClass();
+export default AutoRemediationRuleEngineService;
+//# sourceMappingURL=AutoRemediationRuleEngineService.js.map