mergen-server 1.4.0

package/dist/intelligence/incident-autopilot.js

@@ -0,0 +1,622 @@
+import { randomUUID } from "crypto";
+import { store } from "../sensor/buffer.js";
+import { getRunbookForTag } from "../routes/runbooks.js";
+import { buildCausalChain, fixActionToCommand } from "./causal.js";
+import { getRecords, recordVerdict, classifyVerdict, recordRemediationVerdict, isCorpusSeeded, getRealVerdictCount } from "./calibration.js";
+import { executeRemediation, extractCommand } from "./autonomy.js";
+import { postThreadReply, postApprovalRequest, fetchIncidentChannelContext, postSimpleWebhookNotification } from "./slack.js";
+import { requestApproval } from "./execution-gate.js";
+import { approvalEvents } from "./approval-events.js";
+import { deriveRollback, executeRollback } from "./rollback.js";
+import { captureSnapshot } from "./incident-replay.js";
+import { computeBlastRadius } from "./blast-radius.js";
+import { getExecutionThreshold } from "./threshold-optimizer.js";
+import { isAutopilotEnabled, isShadowMode } from "./execution-mode.js";
+import { incidentStore } from "../sensor/incident-store.js";
+import { getActiveIncident } from "../datadog/incident-state.js";
+import { fetchErrorCountSince, isConfigured as isDatadogConfigured } from "../datadog/client.js";
+import { normalizeRuntimeFactMarkdown, normalizeProcessExits, normalizeSlackContext } from "../sensor/infra-normalizer.js";
+import { getK8sEvents } from "../sensor/k8s-events.js";
+import { dominantOverrideReason } from "./override-corpus.js";
+import { findSimilarDecisions } from "./decision-similarity.js";
+import { blendEscalation } from "./decision-escalation.js";
+import { cacheIncidentResult, getCachedIncidentResult } from "./incident-result-cache.js";
+import { getStores } from "../storage/store-registry.js";
+import { getAutopilotLevel, classifyCommandRisk, autopilotLevelDescription } from "./action-risk.js";
+import { evaluateEnterprisePolicy, loadEnterprisePolicy } from "./enterprise-policy-engine.js";
+import { getStatsForTag } from "./calibration.js";
+import { runAgentPipeline } from "./agent-pipeline.js";
+import { detectCascade } from "./cascade-detector.js";
+import { planningGate } from "./planning-gate.js";
+import { plattScale } from "./platt-scaling.js";
+import { formatValidatedFactsForLLM } from "./llm-spokesperson.js";
+import { serviceGraph } from "../sensor/service-graph.js";
+import { routeReachability } from "../sensor/route-reachability.js";
+import { updateRunbookFromPostmortem } from "./runbook-updater.js";
+import { generatePostmortem } from "./postmortem-store.js";
+import logger from "../sensor/logger.js";
+approvalEvents.on("approval:expired", (pid, text) => {
+  postThreadReply(pid, text).catch((err) => logger.error({ err, pid }, "approval:expired: slack reply failed"));
+});
+async function replyToThread(pid, text) {
+  const ok = await postThreadReply(pid, text);
+  if (!ok) logger.error({ pid }, "incident-autopilot: Slack reply failed \u2014 engineer may not see this update");
+}
+const ANALYSIS_TIMEOUT_MS = 3e4;
+const DEDUP_TTL_MS = 30 * 60 * 1e3;
+const _recentlyTriaged = /* @__PURE__ */ new Map();
+const MAX_CONCURRENT_AUTOPILOT = 3;
+let _activeCalls = 0;
+setInterval(() => {
+  const cutoff = Date.now() - DEDUP_TTL_MS;
+  for (const [k, v] of _recentlyTriaged) if (v < cutoff) _recentlyTriaged.delete(k);
+}, DEDUP_TTL_MS).unref();
+const _stats = {
+  dedupHits: 0,
+  corpusFastPathHits: 0,
+  executedFastPathHits: 0,
+  concurrencyBlocked: 0,
+  tokenBudgetTruncations: 0,
+  concurrentPeak: 0
+};
+function getCostGuardStats() {
+  return { ..._stats, concurrentActive: _activeCalls };
+}
+function _resetTriagedForTesting() {
+  _recentlyTriaged.clear();
+}
+const getAutoExecuteThreshold = () => {
+  const base = getExecutionThreshold();
+  const provisional = isCorpusSeeded() || getRealVerdictCount() < 10;
+  return provisional ? Math.min(0.95, base + 0.05) : base;
+};
+const getBaseUrl = () => (process.env.MERGEN_PUBLIC_URL ?? process.env.MERGEN_BASE_URL ?? "http://127.0.0.1:3000").replace(/\/$/, "");
+function buildVerdictLinks(entry) {
+  const base = getBaseUrl();
+  const approve = `${base}/shadow-report/${entry.id}/verdict?v=approve`;
+  const override = `${base}/shadow-report/${entry.id}/verdict?v=override&reason=on-call-discretion`;
+  return `
+<${approve}|\u2705 Approve>  \xB7  <${override}|\u270B Override>`;
+}
+const TELEMETRY_WAIT_MAX_MS = 1e4;
+const TELEMETRY_POLL_INTERVAL = 250;
+const TELEMETRY_MIN_EVENTS = 3;
+async function waitForTelemetry(firedAt, tenantId) {
+  const deadline = Date.now() + TELEMETRY_WAIT_MAX_MS;
+  while (Date.now() < deadline) {
+    const logs = store.getLogs(TELEMETRY_MIN_EVENTS, void 0, firedAt, tenantId);
+    const net = store.getNetwork(TELEMETRY_MIN_EVENTS, void 0, firedAt, tenantId);
+    if (logs.length + net.length >= TELEMETRY_MIN_EVENTS) return;
+    await new Promise((r) => setTimeout(r, TELEMETRY_POLL_INTERVAL));
+  }
+}
+async function runIncidentAutopilot(opts) {
+  if (!isAutopilotEnabled() && !isShadowMode()) {
+    logger.debug({ service: opts.service }, "incident-autopilot: disabled (set MERGEN_AUTOPILOT=true or MERGEN_SHADOW_MODE=true)");
+    return;
+  }
+  const { service, pid, firedAt, cwd } = opts;
+  logger.info({ service, pid }, "incident-autopilot: starting");
+  if (!isShadowMode()) {
+    const lastTriaged = _recentlyTriaged.get(pid);
+    if (lastTriaged && Date.now() - lastTriaged < DEDUP_TTL_MS) {
+      _stats.dedupHits++;
+      logger.info({ service, pid }, "incident-autopilot: duplicate fingerprint within 30min \u2014 skipping re-analysis");
+      await replyToThread(pid, "_Mergen: same incident fingerprint already triaged within 30 min \u2014 skipping re-analysis._");
+      return;
+    }
+    _recentlyTriaged.set(pid, Date.now());
+    if (_recentlyTriaged.size > 200) {
+      const cutoff = Date.now() - DEDUP_TTL_MS;
+      for (const [k, v] of _recentlyTriaged) if (v < cutoff) _recentlyTriaged.delete(k);
+    }
+  }
+  if (!isShadowMode()) {
+    const cached = getCachedIncidentResult(pid);
+    if (cached?.corpusBlocked) {
+      _stats.corpusFastPathHits++;
+      logger.info({ service, pid, tag: cached.incidentTag }, "incident-autopilot: corpus fast-path \u2014 skipping LLM inference");
+      await replyToThread(
+        pid,
+        `\u26A0\uFE0F _Autopilot: corpus fast-path \u2014 incident pattern \`${cached.incidentTag}\` was previously blocked by the override corpus for \`${service}\`. Awaiting manual action._`
+      );
+      return;
+    }
+    if (cached?.executedCommand) {
+      _stats.executedFastPathHits++;
+      const agoMin = Math.round((Date.now() - cached.cachedAt) / 6e4);
+      logger.info({ service, pid, cmd: cached.executedCommand }, "incident-autopilot: executed fast-path \u2014 fix already applied");
+      await replyToThread(
+        pid,
+        `_Mergen: fix \`${cached.executedCommand}\` was applied for this incident ${agoMin} min ago \u2014 skipping re-analysis. If the issue persists, override the cache via POST /overrides._`
+      );
+      return;
+    }
+  }
+  if (_activeCalls >= MAX_CONCURRENT_AUTOPILOT) {
+    const deadline = Date.now() + 2 * 6e4;
+    while (_activeCalls >= MAX_CONCURRENT_AUTOPILOT && Date.now() < deadline) {
+      await new Promise((r) => setTimeout(r, 1e3));
+    }
+    if (_activeCalls >= MAX_CONCURRENT_AUTOPILOT) {
+      _stats.concurrencyBlocked++;
+      logger.warn({ service, pid, active: _activeCalls }, "incident-autopilot: concurrency cap reached \u2014 aborting");
+      await replyToThread(pid, "_Mergen: too many concurrent analyses in progress \u2014 retry in a few minutes or investigate manually._");
+      return;
+    }
+  }
+  _activeCalls++;
+  if (_activeCalls > _stats.concurrentPeak) _stats.concurrentPeak = _activeCalls;
+  const cascade = detectCascade(service, pid);
+  if (cascade.isCascade) {
+    await replyToThread(pid, cascade.summary);
+  }
+  return _runAutopilotCore(service, pid, firedAt, cwd, opts.tenantId).finally(() => {
+    _activeCalls--;
+  });
+}
+async function _runAutopilotCore(service, pid, firedAt, cwd, tenantId) {
+  await waitForTelemetry(firedAt, tenantId);
+  const logs = store.getLogs(200, void 0, firedAt, tenantId);
+  const network = store.getNetwork(200, void 0, firedAt, tenantId);
+  const contexts = store.getContext(20, firedAt, tenantId);
+  const terminal = store.getTerminalOutput(100, void 0, firedAt, tenantId);
+  const processExits = store.getProcessExits(20, void 0, firedAt, tenantId);
+  const ciEvents = store.getCIEvents(20, void 0, firedAt, tenantId);
+  const deployments = store.getDeployments(10, void 0, firedAt, tenantId);
+  const MAX_BUDGET_LOGS = 60;
+  const MAX_BUDGET_NETWORK = 60;
+  const errorLogs = logs.filter((e) => e.level === "error").slice(-MAX_BUDGET_LOGS);
+  const fillerLogs = logs.filter((e) => e.level !== "error").slice(-(MAX_BUDGET_LOGS - errorLogs.length));
+  const netFails = network.filter((n) => n.status >= 400 || !!n.error).slice(-MAX_BUDGET_NETWORK);
+  const netOk = network.filter((n) => n.status < 400 && !n.error).slice(-(MAX_BUDGET_NETWORK - netFails.length));
+  const budgetedLogs = [...fillerLogs, ...errorLogs];
+  const budgetedNetwork = [...netOk, ...netFails];
+  if (logs.length > MAX_BUDGET_LOGS || network.length > MAX_BUDGET_NETWORK) {
+    _stats.tokenBudgetTruncations++;
+    logger.info(
+      { service, pid, originalLogs: logs.length, budgetedLogs: budgetedLogs.length, originalNetwork: network.length, budgetedNetwork: budgetedNetwork.length },
+      "incident-autopilot: token budget truncated telemetry"
+    );
+  }
+  const errorCount = logs.filter((e) => e.level === "error").length;
+  const netErrors = network.filter((n) => n.status >= 400 || !!n.error).length;
+  const slackContextText = await fetchIncidentChannelContext(firedAt).catch(() => null);
+  const activeIncident = getActiveIncident();
+  const runtimeFactMarkdown = activeIncident?.runtimeFact ?? null;
+  const infraEvents = [
+    ...runtimeFactMarkdown ? normalizeRuntimeFactMarkdown(runtimeFactMarkdown, service, firedAt) : [],
+    ...normalizeProcessExits(processExits),
+    ...getK8sEvents(firedAt),
+    ...slackContextText ? normalizeSlackContext(slackContextText, service, firedAt) : []
+  ];
+  if (slackContextText) {
+    logger.info({ service, pid, chars: slackContextText.length }, "incident-autopilot: slack channel context added to evidence");
+  }
+  const hasAnySignal = errorCount > 0 || netErrors > 0 || infraEvents.length > 0;
+  if (!hasAnySignal) {
+    logger.info({ service, pid }, "incident-autopilot: no signals found \u2014 skipping");
+    await replyToThread(pid, "_Mergen autopilot: no errors or infra signals found in buffer \u2014 manual investigation required._");
+    return;
+  }
+  if (runtimeFactMarkdown) {
+    const ageMin = Math.round((Date.now() - firedAt) / 6e4);
+    await replyToThread(
+      pid,
+      [
+        `\u{1F4E1} *Mergen Autopilot \u2014 Incident Context* (${ageMin}m after alert)`,
+        "",
+        runtimeFactMarkdown
+      ].join("\n")
+    );
+  }
+  let causal = null;
+  try {
+    causal = await Promise.race([
+      buildCausalChain(budgetedLogs, budgetedNetwork, contexts, firedAt, terminal, processExits, ciEvents, deployments, infraEvents),
+      new Promise(
+        (_, reject) => setTimeout(() => reject(new Error("analysis timeout")), ANALYSIS_TIMEOUT_MS)
+      )
+    ]);
+  } catch (err) {
+    logger.warn({ err, service, pid }, "incident-autopilot: causal analysis failed \u2014 posting raw telemetry");
+    const topErrors = logs.filter((e) => e.level === "error").slice(0, 5);
+    const topNetFails = network.filter((n) => n.status >= 400 || !!n.error).slice(0, 5);
+    await replyToThread(pid, [
+      `\u26A1 *Mergen \u2014 Raw Telemetry Snapshot* (analysis unavailable)`,
+      `*${errorCount} console errors, ${netErrors} network failures* in window`,
+      topErrors.length > 0 ? `*Top errors:*
+${topErrors.map((e) => `\u2022 ${String(e.args?.[0] ?? "").slice(0, 120)}`).join("\n")}` : "",
+      topNetFails.length > 0 ? `*Network failures:*
+${topNetFails.map((n) => `\u2022 ${n.method} ${n.url} \u2192 ${n.status}`).join("\n")}` : "",
+      `_Manual investigation required \u2014 AI analysis unavailable._`
+    ].filter(Boolean).join("\n"));
+    return;
+  }
+  if (serviceGraph.size > 0) {
+    const callers = new Set(serviceGraph.getCallers(service));
+    const callees = new Set(serviceGraph.getCallees(service));
+    const graphNeighbours = /* @__PURE__ */ new Set([...callers, ...callees, service]);
+    for (const hyp of causal.hypotheses) {
+      const isUpstream = /upstream|cascade|spike|flood|overload/i.test(hyp.summary);
+      if (isUpstream && callers.size === 0) {
+        hyp.confidenceScore = Math.max(0, hyp.confidenceScore * 0.7);
+      } else if (isUpstream && callers.size > 0) {
+        hyp.confidenceScore = Math.min(1, hyp.confidenceScore * 1.1);
+      }
+      const isExternal = /database|db|cache|redis|postgres|mysql|mongo|timeout/i.test(hyp.summary);
+      if (isExternal && callees.size === 0) {
+        hyp.confidenceScore = Math.max(0, hyp.confidenceScore * 0.75);
+      }
+      const routeMatch = hyp.summary.match(/\/(api|v\d|webhooks|auth|admin|internal)[^\s"')>]*/i);
+      if (routeMatch && routeReachability.size > 0 && !routeReachability.isReachable(routeMatch[0])) {
+        hyp.confidenceScore = Math.max(0, hyp.confidenceScore * 0.7);
+      }
+    }
+    logger.debug(
+      { service, callers: callers.size, callees: callees.size, routes: routeReachability.size, hypotheses: causal.hypotheses.length },
+      "incident-autopilot: topology filter applied"
+    );
+  }
+  const plattAdjusted = /* @__PURE__ */ new Map();
+  for (const hyp of causal.hypotheses) {
+    const { calibrated, source } = plattScale(hyp.confidenceScore, hyp.tag);
+    if (source !== "raw") plattAdjusted.set(hyp, calibrated);
+  }
+  causal.hypotheses.sort((a, b) => {
+    const aScore = plattAdjusted.get(a) ?? a.confidenceScore;
+    const bScore = plattAdjusted.get(b) ?? b.confidenceScore;
+    return bScore - aScore;
+  });
+  const topHyp = causal.hypotheses[0];
+  captureSnapshot({
+    pid,
+    capturedAt: Date.now(),
+    firedAt,
+    logs,
+    network,
+    contexts,
+    terminal,
+    processExits,
+    ciEvents,
+    deployments,
+    infraEvents,
+    originalTag: topHyp?.tag ?? null,
+    originalConfidenceScore: topHyp?.confidenceScore ?? null,
+    originalFixHint: topHyp?.fixHint ?? null
+  });
+  if (topHyp && activeIncident?.implicatedFile && topHyp.fixHint) {
+    const loc = activeIncident.implicatedLine ? `${activeIncident.implicatedFile}:${activeIncident.implicatedLine}` : activeIncident.implicatedFile;
+    if (!topHyp.fixHint.includes(activeIncident.implicatedFile)) {
+      topHyp.fixHint = `${topHyp.fixHint}
+Failing location: \`${loc}\``;
+    }
+  }
+  if (!topHyp) {
+    logger.info({ service, pid }, "incident-autopilot: no hypothesis generated");
+    await replyToThread(pid, `_Mergen autopilot: analyzed ${errorCount} console errors and ${netErrors} network errors \u2014 no actionable root cause identified._`);
+    return;
+  }
+  const rawScore = topHyp.confidenceScore ?? 0;
+  const calibratedScore = plattAdjusted.get(topHyp) ?? rawScore;
+  const pct = Math.round(calibratedScore * 100);
+  const calStats = getStatsForTag(topHyp.tag);
+  const calibrationLabel = calStats?.isEmpirical ? `calibrated \u2014 ${calStats.verdicts} verdicts` : "estimated \u2014 self-calibrates with use";
+  const now = /* @__PURE__ */ new Date();
+  const pipeline = runAgentPipeline(causal, {
+    service,
+    executionThreshold: getAutoExecuteThreshold(),
+    service_time: { dayOfWeek: now.getUTCDay(), hourOfDay: now.getUTCHours() }
+  });
+  const pipelineLine = isAutopilotEnabled() || isShadowMode() ? `*Pipeline:* ${pipeline.stages.map((s) => {
+    const icon = s.status === "pass" ? "\u2705" : s.status === "warn" ? "\u26A0\uFE0F" : "\u{1F6AB}";
+    return `${icon} ${s.name}`;
+  }).join(" \xB7 ")}` : "";
+  const preShadowId = isShadowMode() && topHyp.pid ? randomUUID() : void 0;
+  const preShadowVerdictLinks = preShadowId ? `
+${buildVerdictLinks({ id: preShadowId })}` : "";
+  const diagMsg = [
+    `\u{1F50D} *Mergen Autopilot \u2014 Root Cause Analysis*`,
+    `*Hypothesis:* ${topHyp.summary}`,
+    `*Confidence:* ${topHyp.confidence} (${pct}%) [${calibrationLabel}]`,
+    topHyp.causalPath.length > 0 ? `*Causal path:*
+${topHyp.causalPath.map((s, i) => `${i + 1}. ${s}`).join("\n")}` : "",
+    topHyp.evidence.length > 0 ? `*Evidence:*
+${topHyp.evidence.map((e) => `\u2022 ${e}`).join("\n")}` : "",
+    topHyp.fixHint ? `*Fix:* ${topHyp.fixHint}` : "",
+    pipelineLine
+  ].filter(Boolean).join("\n") + preShadowVerdictLinks;
+  await replyToThread(pid, diagMsg);
+  logger.info({ service, pid, confidence: pct, hypothesis: topHyp.tag }, "incident-autopilot: diagnosis posted");
+  const webhookMsg = `\u{1F4E2} *Triage complete* for *${service}*. Hypothesis: *${topHyp.summary}* (${pct}% confidence). View unified timeline in Cursor/Claude Code.`;
+  void postSimpleWebhookNotification(service, webhookMsg);
+  const approvedRunbook = getRunbookForTag(topHyp.tag);
+  const runbookCommand = approvedRunbook ? approvedRunbook.steps.join(" && ") : null;
+  if (approvedRunbook) {
+    logger.info({ id: approvedRunbook.id, name: approvedRunbook.name, tag: topHyp.tag }, "incident-autopilot: using pre-approved runbook");
+  }
+  const command = runbookCommand ?? pipeline.plan?.command ?? (topHyp.fixAction ? fixActionToCommand(topHyp.fixAction) : null) ?? (topHyp.fixHint ? extractCommand(topHyp.fixHint) : null);
+  const execConfidence = topHyp.remediationConfidence ?? topHyp.confidenceScore ?? 0;
+  const execPct = Math.round(execConfidence * 100);
+  const gate = planningGate(topHyp, service, getAutoExecuteThreshold());
+  const gateDenied = !gate.execute;
+  const runtimeFact = activeIncident?.runtimeFact ?? null;
+  const brief = formatValidatedFactsForLLM(topHyp, service, gate, errorCount, runtimeFact, calibratedScore);
+  logger.debug({ service, pid, tokens: brief.estimatedTokens }, "incident-autopilot: LLM brief assembled");
+  if (gate.adjustedConfidence !== execConfidence) {
+    logger.info(
+      { service, pid, raw: execPct, adjusted: Math.round(gate.adjustedConfidence * 100), blastRisk: gate.signals.blastRisk, classifier: gate.signals.classifierScore },
+      "incident-autopilot: planning gate adjusted confidence"
+    );
+  }
+  const corpusBlocked = pipeline.critique?.corpusConflict ?? false;
+  const levelBlocked = pipeline.critique?.levelConflict ?? false;
+  const pipelineBlocked = pipeline.verdict === "block";
+  const pipelineReview = pipeline.verdict === "review";
+  let escalationBlocked = false;
+  let escalation = null;
+  if (command) {
+    try {
+      const [historicalOverrides, historicalShadowEntries] = await Promise.all([
+        getStores().overrides.getAllOverrides(tenantId),
+        getStores().shadowLog.getShadowEntries(tenantId)
+      ]);
+      const similarity = findSimilarDecisions(command, service, historicalOverrides, historicalShadowEntries);
+      escalation = blendEscalation({ similarity, calibration: getStatsForTag(topHyp.tag) ?? null });
+      escalationBlocked = escalation.recommendation === "hold";
+    } catch (err) {
+      logger.warn({ err, service, pid }, "incident-autopilot: decision-escalation lookup failed");
+    }
+  }
+  if (!command || pipelineBlocked || corpusBlocked || escalationBlocked || isShadowMode() || gateDenied || !pipelineReview && execConfidence < getAutoExecuteThreshold()) {
+    let skipReason;
+    let slackReason;
+    if (isShadowMode()) {
+      skipReason = "autopilot-disabled";
+      slackReason = `shadow mode \u2014 would execute \`${command ?? "no command"}\` (remediation: ${execPct}%)`;
+    } else if (!command) {
+      skipReason = "no-command";
+      slackReason = "no executable command in fixHint";
+    } else if (pipelineBlocked) {
+      skipReason = pipeline.critique?.levelConflict ? "level-restricted" : pipeline.critique?.corpusConflict ? "override-corpus" : "pipeline-block";
+      slackReason = pipeline.blockReason ?? "Governance pipeline blocked execution";
+    } else if (corpusBlocked) {
+      skipReason = "override-corpus";
+      const corpusReason = dominantOverrideReason(topHyp.tag, service);
+      slackReason = `override corpus: this action has been overridden before for \`${service}\` (reason: ${corpusReason ?? "unknown"})`;
+    } else if (escalationBlocked) {
+      skipReason = "similar-incident-history";
+      slackReason = `historical decision memory: ${escalation?.reasons.join(" ") ?? "similar past actions and calibration confidence flagged this for review"}`;
+    } else if (gateDenied) {
+      skipReason = "planning-gate";
+      slackReason = `planning gate: ${gate.reason}`;
+    } else if (levelBlocked) {
+      skipReason = "level-restricted";
+      const commandTier2 = classifyCommandRisk(command);
+      const autopilotLevel = getAutopilotLevel();
+      slackReason = `autopilot level \`${autopilotLevel}\` permits ${autopilotLevelDescription(autopilotLevel)} \u2014 this command is \`${commandTier2}\` tier. Set MERGEN_AUTOPILOT_LEVEL=full to enable.`;
+    } else if (topHyp.remediationConfidence !== void 0 && topHyp.remediationConfidence < getAutoExecuteThreshold()) {
+      skipReason = "remediation-below-threshold";
+      slackReason = `remediation confidence ${execPct}% below 85% threshold (diagnosis: ${pct}%)`;
+    } else {
+      skipReason = "confidence-below-threshold";
+      slackReason = `confidence ${execPct}% below 85% threshold`;
+    }
+    let shadowEntry;
+    if (topHyp.pid) {
+      shadowEntry = await getStores().shadowLog.recordShadow({
+        ...preShadowId ? { id: preShadowId } : {},
+        pid: topHyp.pid,
+        incidentTag: topHyp.tag,
+        service,
+        command,
+        diagnosisConfidence: topHyp.confidenceScore ?? 0,
+        remediationConfidence: execConfidence,
+        wouldHaveExecuted: !!command && execConfidence >= getAutoExecuteThreshold() && !corpusBlocked && !escalationBlocked,
+        skipReason: skipReason === "pipeline-block" ? "confidence-below-threshold" : skipReason,
+        firedAt
+      }, tenantId);
+    }
+    logger.info({ service, pid, skipReason, diagPct: pct, execPct, pipelineVerdict: pipeline.verdict }, "incident-autopilot: skipping auto-execute");
+    if (skipReason === "override-corpus") {
+      await getStores().blunders.record({ blunderType: "override_corpus_block", command, blockReason: slackReason, service, tag: topHyp.tag, actor: "autopilot", pid: topHyp.pid ?? pid, confidenceScore: execConfidence });
+      cacheIncidentResult({ fingerprint: pid, service, incidentTag: topHyp.tag, corpusBlocked: true, blockReason: slackReason, executedCommand: null });
+    } else if (skipReason === "pipeline-block" || skipReason === "level-restricted") {
+      await getStores().blunders.record({ blunderType: "pipeline_block", command, blockReason: slackReason, service, tag: topHyp.tag, actor: "autopilot", pid: topHyp.pid ?? pid, confidenceScore: execConfidence });
+    } else if (skipReason === "planning-gate") {
+      await getStores().blunders.record({ blunderType: "planning_gate_block", command, blockReason: slackReason, service, tag: topHyp.tag, actor: "autopilot", pid: topHyp.pid ?? pid, confidenceScore: execConfidence });
+    }
+    const icon = isShadowMode() ? "\u{1F441}\uFE0F" : "\u26A0\uFE0F";
+    const verdictLinks = isShadowMode() && shadowEntry ? buildVerdictLinks(shadowEntry) : "";
+    await replyToThread(pid, `${icon} _Autopilot: ${slackReason}. Awaiting manual action._${verdictLinks}`);
+    return;
+  }
+  let hasHighRiskPolicy = false;
+  try {
+    const evalRes = evaluateEnterprisePolicy({
+      files: [],
+      commands: [command],
+      actor: "autopilot",
+      service
+    });
+    if (evalRes.triggeredRules.length > 0) {
+      const policyConfig = loadEnterprisePolicy();
+      const triggeredRulesWithRisk = policyConfig.rules.filter(
+        (r) => evalRes.triggeredRules.includes(r.id) && (r.riskTier === "medium" || r.riskTier === "high")
+      );
+      if (triggeredRulesWithRisk.length > 0) {
+        hasHighRiskPolicy = true;
+        logger.info({ triggeredRules: triggeredRulesWithRisk.map((r) => r.id) }, "incident-autopilot: command matched a medium/high risk policy rule");
+      }
+    }
+  } catch (err) {
+    logger.error({ err }, "incident-autopilot: failed to evaluate enterprise policy risk rules");
+  }
+  const commandTier = classifyCommandRisk(command);
+  const blastRadius = computeBlastRadius(command, { service });
+  if (hasHighRiskPolicy || commandTier !== "restart" && getAutopilotLevel() !== "full") {
+    logger.info({ service, pid, command, commandTier, blastScope: blastRadius.scope, hasHighRiskPolicy }, "incident-autopilot: routing through approval gate");
+    const approvalPosted = await postApprovalRequest(pid, command, commandTier, execConfidence, blastRadius);
+    if (!approvalPosted) {
+      logger.error({ service, pid, command }, "incident-autopilot: aborting approval gate \u2014 Slack notification failed, engineer cannot approve");
+      await getStores().blunders.record({ blunderType: "pipeline_block", command, blockReason: "Slack unavailable \u2014 approval block could not be delivered", service, tag: topHyp.tag, actor: "autopilot", pid: topHyp.pid ?? pid, confidenceScore: execConfidence });
+      return;
+    }
+    requestApproval({ pid, command, tier: commandTier, service, remediationConfidence: execConfidence, cwd, blastRadius });
+    void getStores().shadowLog.recordShadow({
+      pid: topHyp.pid ?? pid,
+      incidentTag: topHyp.tag,
+      service,
+      command,
+      diagnosisConfidence: topHyp.confidenceScore ?? 0,
+      remediationConfidence: execConfidence,
+      wouldHaveExecuted: true,
+      skipReason: "level-restricted",
+      firedAt
+    }, tenantId);
+    return;
+  }
+  logger.info({ service, pid, command }, "incident-autopilot: executing fix");
+  const execResult = await executeRemediation(command, { cwd, actor: "autopilot" });
+  if (execResult.blocked) {
+    logger.warn({ service, pid, reason: execResult.blockReason }, "incident-autopilot: fix blocked by safety filter");
+    await replyToThread(pid, `\u{1F6AB} *Fix blocked by safety filter*: ${execResult.blockReason}
+Apply manually.`);
+    const execBlunderType = typeof execResult.blockReason === "string" && /inject/i.test(execResult.blockReason) ? "injection_attempt" : "allowlist_block";
+    await getStores().blunders.record({ blunderType: execBlunderType, command, blockReason: execResult.blockReason ?? "", service, tag: topHyp.tag, actor: "autopilot", pid: topHyp.pid ?? pid, confidenceScore: execConfidence });
+    void getStores().shadowLog.recordShadow({
+      pid: topHyp.pid ?? pid,
+      incidentTag: topHyp.tag,
+      service,
+      command,
+      diagnosisConfidence: topHyp.confidenceScore ?? 0,
+      remediationConfidence: execConfidence,
+      wouldHaveExecuted: true,
+      skipReason: "blocked-by-safety-filter",
+      firedAt
+    }, tenantId);
+    return;
+  }
+  if (!execResult.ok) {
+    logger.warn({ service, pid, exitCode: execResult.exitCode }, "incident-autopilot: fix command failed");
+    await replyToThread(pid, `\u274C *Fix command failed* (exit ${execResult.exitCode})
+${execResult.stderr.slice(0, 500)}`);
+    void getStores().shadowLog.recordShadow({
+      pid: topHyp.pid ?? pid,
+      incidentTag: topHyp.tag,
+      service,
+      command,
+      diagnosisConfidence: topHyp.confidenceScore ?? 0,
+      remediationConfidence: execConfidence,
+      wouldHaveExecuted: true,
+      skipReason: "executed-failure",
+      firedAt
+    }, tenantId);
+    return;
+  }
+  const fixAppliedAt = Date.now();
+  await replyToThread(pid, `\u2699\uFE0F \`${command}\` executed (${execResult.durationMs}ms) \u2014 validating\u2026`);
+  void getStores().shadowLog.recordShadow({
+    pid: topHyp.pid ?? pid,
+    incidentTag: topHyp.tag,
+    service,
+    command,
+    diagnosisConfidence: topHyp.confidenceScore ?? 0,
+    remediationConfidence: execConfidence,
+    wouldHaveExecuted: true,
+    skipReason: "executed",
+    firedAt,
+    ...approvedRunbook ? { runbookId: approvedRunbook.id } : {}
+  }, tenantId);
+  cacheIncidentResult({ fingerprint: pid, service, incidentTag: topHyp.tag, corpusBlocked: false, blockReason: null, executedCommand: command });
+  await new Promise((r) => setTimeout(r, 5e3));
+  let afterCount;
+  const beforeCount = errorCount + netErrors;
+  const ddErrorCount = isDatadogConfigured() ? await fetchErrorCountSince(service, 2) : null;
+  if (ddErrorCount !== null) {
+    afterCount = ddErrorCount;
+    logger.info({ service, pid, ddErrorCount }, "incident-autopilot: validation via Datadog");
+  } else {
+    const logsAfter = store.getLogs(200, "error", fixAppliedAt, tenantId);
+    const netAfter = store.getNetwork(200, void 0, fixAppliedAt, tenantId).filter((n) => n.status >= 400 || !!n.error);
+    afterCount = logsAfter.length + netAfter.length;
+  }
+  const verdict = classifyVerdict(beforeCount, afterCount);
+  if (topHyp.pid) {
+    const existing = getRecords().find((r) => r.pid === topHyp.pid);
+    if (existing && !existing.verdict) recordVerdict(topHyp.pid, verdict);
+    recordRemediationVerdict(topHyp.pid, verdict);
+    const resolvedAt = Date.now();
+    if (verdict === "correct") {
+      incidentStore.upsert(topHyp.pid, {
+        status: "resolved",
+        resolvedAt,
+        resolvedAutonomously: true,
+        causallyCorrect: true
+        // error rate dropped AND diagnosis confirmed
+      });
+    } else {
+      incidentStore.upsert(topHyp.pid, {
+        status: "resolved",
+        resolvedAt,
+        resolvedAutonomously: true,
+        causallyCorrect: false
+      });
+    }
+    try {
+      const mttrMs = resolvedAt - firedAt;
+      const pm = generatePostmortem({
+        pid: topHyp.pid,
+        tag: topHyp.tag,
+        service,
+        rootCause: topHyp.summary,
+        fixCommand: command,
+        confidence: topHyp.confidenceScore ?? 0,
+        mttrMs,
+        resolvedAutonomously: true,
+        causallyCorrect: verdict === "correct",
+        evidence: topHyp.evidence,
+        fixHint: topHyp.fixHint ?? null
+      });
+      setImmediate(() => updateRunbookFromPostmortem(pm));
+    } catch (err) {
+      logger.warn({ err, pid: topHyp.pid }, "incident-autopilot: postmortem generation failed");
+    }
+  }
+  const statusLabel = afterCount === 0 ? "RESOLVED" : afterCount < beforeCount ? "PARTIAL" : afterCount > beforeCount ? "REGRESSED" : "UNRESOLVED";
+  const statusIcon = afterCount === 0 ? "\u2705" : afterCount < beforeCount ? "\u26A0\uFE0F" : "\u274C";
+  await replyToThread(
+    pid,
+    `${statusIcon} *${statusLabel}* \u2014 ${afterCount} errors after fix (was ${beforeCount})`
+  );
+  if (statusLabel === "REGRESSED") {
+    const rollback = deriveRollback(command, execResult.stdout);
+    if (rollback.type === "command") {
+      await replyToThread(pid, `\u21A9\uFE0F _REGRESSED detected \u2014 attempting auto-rollback\u2026_`);
+      const rb = await executeRollback(rollback, { cwd, actor: "autopilot-rollback" });
+      await replyToThread(
+        pid,
+        rb.ok ? `\u21A9\uFE0F *Rollback succeeded:* \`${rb.message}\`` : `\u{1F534} *Rollback failed:* ${rb.message} \u2014 manual intervention required`
+      );
+    } else {
+      await replyToThread(pid, `\u26A0\uFE0F _Auto-rollback not available: ${rollback.reason}. Manual revert required._`);
+    }
+  }
+  logger.info({ service, pid, statusLabel, afterCount, beforeCount }, "incident-autopilot: complete");
+}
+async function enqueueAutopilot(opts) {
+  const useBullMQ = (process.env.MERGEN_CLOUD_MODE === "true" || process.env.MERGEN_BULLMQ === "true") && process.env.MERGEN_REDIS_URL;
+  if (useBullMQ) {
+    const { getAutopilotQueue } = await import("../workers/queues.js");
+    await getAutopilotQueue().add("triage-incident", opts, { jobId: opts.pid });
+  } else {
+    void runIncidentAutopilot(opts);
+  }
+}
+export {
+  _resetTriagedForTesting,
+  enqueueAutopilot,
+  getCostGuardStats,
+  runIncidentAutopilot,
+  runIncidentAutopilot as runIncidentAutopilotLocal
+};