mergen-server 1.4.0

package/dist/routes/pagerduty.js

@@ -0,0 +1,258 @@
+import crypto from "crypto";
+import { Router } from "express";
+import { z } from "zod";
+import { CLOUD_MODE } from "../sensor/cloud-auth.js";
+import { fetchLatestErrorTrace, isConfigured } from "../datadog/client.js";
+import { compact } from "../datadog/compactor.js";
+import { setActiveIncident } from "../datadog/incident-state.js";
+import { memoryStore } from "../datadog/memory-store.js";
+import { fingerprintFromFact } from "../datadog/fingerprinter.js";
+import { computeBlameAttribution } from "../datadog/blame-attribution.js";
+import { postIncidentAlert } from "../intelligence/slack.js";
+import { store } from "../sensor/buffer.js";
+import { runIncidentAutopilot } from "../intelligence/incident-autopilot.js";
+import { getRecords, recordVerdict } from "../intelligence/calibration.js";
+import { recordOverride } from "../intelligence/override-corpus.js";
+import logger from "../sensor/logger.js";
+const DASHBOARD_URL = process.env.MERGEN_DASHBOARD_URL ?? "http://127.0.0.1:3000";
+const PD_WEBHOOK_SECRET = process.env.MERGEN_PAGERDUTY_SECRET;
+if (PD_WEBHOOK_SECRET) {
+  logger.info("pagerduty: webhook signature verification enabled");
+} else if (CLOUD_MODE) {
+  logger.error(
+    "pagerduty: MERGEN_PAGERDUTY_SECRET is not set in cloud mode \u2014 all webhook requests will be rejected. Set MERGEN_PAGERDUTY_SECRET to the signing secret from your PagerDuty webhook config."
+  );
+} else {
+  logger.warn(
+    "pagerduty: MERGEN_PAGERDUTY_SECRET is not set \u2014 webhook signature verification is DISABLED. Any caller can forge PagerDuty incident.triggered events and trigger autonomous execution. Set MERGEN_PAGERDUTY_SECRET to the signing secret from your PagerDuty webhook config."
+  );
+}
+function verifyPagerDutySignature(rawBody, header) {
+  if (!PD_WEBHOOK_SECRET) {
+    if (process.env.MERGEN_AUTOPILOT === "true") return false;
+    if (CLOUD_MODE) return false;
+    return true;
+  }
+  if (!header) return false;
+  const expected = "v1=" + crypto.createHmac("sha256", PD_WEBHOOK_SECRET).update(rawBody).digest("hex");
+  return header.split(",").some((sig) => {
+    const s = sig.trim();
+    try {
+      return s.length === expected.length && crypto.timingSafeEqual(Buffer.from(s), Buffer.from(expected));
+    } catch {
+      return false;
+    }
+  });
+}
+const PdMessageSchema = z.object({
+  messages: z.array(
+    z.object({
+      event: z.object({
+        event_type: z.string(),
+        data: z.object({
+          id: z.string(),
+          summary: z.string().optional(),
+          html_url: z.string().optional(),
+          created_at: z.string().optional(),
+          resolved_at: z.string().optional(),
+          service: z.object({ summary: z.string() }).optional()
+        })
+      })
+    })
+  )
+});
+function createPagerDutyRouter() {
+  const router = Router();
+  function handleWebhook(req, res) {
+    const tenantId = req.params.tenantId;
+    if (CLOUD_MODE && !tenantId) {
+      res.status(400).json({
+        error: "Cloud mode requires a tenant-scoped webhook URL: /webhooks/pagerduty/<tenantId>",
+        docs: "https://github.com/omertt27/Mergen/blob/main/INSTALL.md#pagerduty-cloud-mode"
+      });
+      return;
+    }
+    const MAX_BODY_BYTES = 2 * 1024 * 1024;
+    const chunks = [];
+    let totalBytes = 0;
+    req.on("data", (chunk) => {
+      totalBytes += chunk.length;
+      if (totalBytes > MAX_BODY_BYTES) {
+        res.status(413).json({ error: "payload too large" });
+        req.destroy();
+        return;
+      }
+      chunks.push(chunk);
+    });
+    req.on("end", () => {
+      if (res.headersSent) return;
+      const rawBody = Buffer.concat(chunks);
+      const signature = req.headers["x-pagerduty-signature"];
+      if (!verifyPagerDutySignature(rawBody, signature)) {
+        res.status(401).json({ error: "invalid x-pagerduty-signature" });
+        return;
+      }
+      let body;
+      try {
+        body = JSON.parse(rawBody.toString("utf8"));
+      } catch {
+        res.status(400).json({ error: "malformed JSON" });
+        return;
+      }
+      const parsed = PdMessageSchema.safeParse(body);
+      if (!parsed.success) {
+        res.status(400).json({ error: "invalid PagerDuty payload" });
+        return;
+      }
+      processMessages(parsed.data.messages, res, tenantId);
+    });
+  }
+  router.post("/webhooks/pagerduty", handleWebhook);
+  router.post("/webhooks/pagerduty/:tenantId", handleWebhook);
+  function processMessages(messages, res, tenantId) {
+    for (const msg of messages) {
+      const { event_type, data } = msg.event;
+      if (event_type === "incident.resolved") {
+        const resolvedAt = data.resolved_at ? new Date(data.resolved_at).getTime() : Date.now();
+        const openRecord = memoryStore.listOpen().find((r) => r.pdIncidentId === data.id);
+        if (openRecord) {
+          const mttrMs = resolvedAt - openRecord.firedAt;
+          const QUICK_RESOLVE_MS = 45 * 60 * 1e3;
+          if (mttrMs < QUICK_RESOLVE_MS && openRecord.resolutionType === "unknown") {
+            const fingerprint = openRecord.fingerprint;
+            const unverifiedRecords = getRecords().filter(
+              (r) => r.pid === fingerprint && r.verdict === null
+            );
+            for (const cr of unverifiedRecords) {
+              recordVerdict(cr.pid, "correct", `pd-resolved-in-${Math.round(mttrMs / 6e4)}m`);
+            }
+          }
+        }
+        memoryStore.closeIncident({ pdIncidentId: data.id, resolvedAt });
+        const resolvedService = openRecord?.service ?? data.service?.summary ?? "unknown";
+        const alertSummary = openRecord?.pdAlertTitle ?? data.summary ?? "";
+        if (resolvedService !== "unknown" && alertSummary && PD_WEBHOOK_SECRET) {
+          try {
+            recordOverride({
+              incidentTag: "pagerduty_incident",
+              proposedCommand: alertSummary.slice(0, 500),
+              overrideReason: "on-call-discretion",
+              note: `PagerDuty auto-seed: ${alertSummary.slice(0, 200)}`,
+              rationale: "Auto-seeded from resolved PagerDuty incident \u2014 reviewed by on-call.",
+              service: resolvedService,
+              environment: "production",
+              actor: "pagerduty-auto-seed",
+              source: "team"
+            });
+            logger.info({ pdId: data.id, service: resolvedService }, "pagerduty: auto-seeded override corpus from resolved incident");
+          } catch {
+          }
+        }
+        logger.info({ pdId: data.id }, "pagerduty incident resolved");
+        continue;
+      }
+      if (event_type !== "incident.triggered") continue;
+      const service = data.service?.summary ?? "unknown";
+      const alertTitle = data.summary ?? `Incident ${data.id}`;
+      const alertUrl = data.html_url;
+      const firedAt = data.created_at ? new Date(data.created_at).getTime() : Date.now();
+      logger.info({ service, alertTitle }, "pagerduty incident triggered");
+      setActiveIncident({ service, traceId: "", alertTitle, alertUrl, firedAt });
+      if (isConfigured()) {
+        void (async () => {
+          try {
+            const result = await fetchLatestErrorTrace(service, 10);
+            if (!result) {
+              memoryStore.openIncident({
+                fingerprint: "unknown",
+                service,
+                endpoint: "unknown",
+                errorType: "",
+                errorMessage: alertTitle,
+                pdIncidentId: data.id,
+                pdAlertTitle: alertTitle,
+                pdAlertUrl: alertUrl,
+                traceId: "",
+                firedAt
+              });
+              return;
+            }
+            const to = /* @__PURE__ */ new Date();
+            const from = new Date(to.getTime() - 10 * 60 * 1e3);
+            const { fact } = await compact({
+              spans: result.spans,
+              traceId: result.traceId,
+              timeWindow: { from, to }
+            });
+            const fingerprint = fingerprintFromFact(fact);
+            const deploys = store.getDeployments(50);
+            const blame = computeBlameAttribution({
+              implicatedFile: fact.failingFile ?? null,
+              deployedSha: fact.deployedSha ?? null,
+              firedAt,
+              candidates: deploys
+            });
+            if (blame) {
+              logger.info(
+                { confidence: blame.confidence, label: blame.confidenceLabel, sha: blame.topCandidate?.sha },
+                "blame attribution computed"
+              );
+            }
+            memoryStore.openIncident({
+              fingerprint,
+              service: fact.service,
+              endpoint: fact.endpoint,
+              errorType: fact.errorMessage.split(":")[0],
+              errorMessage: fact.errorMessage,
+              implicatedFile: fact.failingFile,
+              implicatedLine: fact.failingLine,
+              deployedSha: fact.deployedSha,
+              pdIncidentId: data.id,
+              pdAlertTitle: alertTitle,
+              pdAlertUrl: alertUrl,
+              traceId: fact.traceId,
+              rawFact: fact.markdown,
+              firedAt,
+              attributionConfidence: blame?.confidence,
+              attributionSha: blame?.topCandidate?.sha
+            });
+            setActiveIncident({
+              service,
+              traceId: result.traceId,
+              alertTitle,
+              alertUrl,
+              firedAt,
+              runtimeFact: fact.markdown,
+              implicatedFile: fact.failingFile ?? null,
+              implicatedLine: fact.failingLine ?? null,
+              blameAttribution: blame
+            });
+            const blastRadius = store.getBlastRadius({ since: firedAt });
+            const openIncident = memoryStore.listOpen()[0];
+            void postIncidentAlert({
+              alertTitle,
+              service,
+              firedAt,
+              incidentId: openIncident?.id,
+              pid: fingerprint,
+              // use fingerprint as pid for thread ownership
+              pdUrl: alertUrl,
+              blame,
+              blastRadius,
+              dashboardUrl: DASHBOARD_URL
+            });
+            void runIncidentAutopilot({ service, pid: fingerprint, firedAt, tenantId });
+            logger.info({ service, traceId: result.traceId, fingerprint }, "runtime fact pre-computed");
+          } catch (err) {
+            logger.warn({ err }, "failed to pre-compute runtime fact");
+          }
+        })();
+      }
+    }
+    res.json({ status: "accepted" });
+  }
+  return router;
+}
+export {
+  createPagerDutyRouter
+};