mergen-server 1.4.0

package/dist/intelligence/incident-replay.js

@@ -0,0 +1,162 @@
+import fs from "fs";
+import path from "path";
+import { DATA_DIR } from "../sensor/paths.js";
+import { buildCausalChain } from "./causal.js";
+import logger from "../sensor/logger.js";
+const REPLAY_DIR = path.join(DATA_DIR, "replay-snapshots");
+const MAX_SNAPSHOTS = 500;
+function captureSnapshot(snapshot) {
+  try {
+    fs.mkdirSync(REPLAY_DIR, { recursive: true });
+    fs.writeFileSync(
+      path.join(REPLAY_DIR, `${snapshot.pid}.json`),
+      JSON.stringify(snapshot),
+      "utf8"
+    );
+    const files = fs.readdirSync(REPLAY_DIR).filter((f) => f.endsWith(".json"));
+    if (files.length > MAX_SNAPSHOTS) {
+      const withMtime = files.map((f) => {
+        try {
+          return { f, mtime: fs.statSync(path.join(REPLAY_DIR, f)).mtimeMs };
+        } catch {
+          return { f, mtime: 0 };
+        }
+      });
+      withMtime.sort((a, b) => a.mtime - b.mtime);
+      for (const { f } of withMtime.slice(0, files.length - MAX_SNAPSHOTS)) {
+        try {
+          fs.unlinkSync(path.join(REPLAY_DIR, f));
+        } catch {
+        }
+      }
+    }
+  } catch (err) {
+    logger.warn({ err, pid: snapshot.pid }, "incident-replay: failed to capture snapshot");
+  }
+}
+async function replayIncident(pid) {
+  const file = path.join(REPLAY_DIR, `${pid}.json`);
+  if (!fs.existsSync(file)) return null;
+  let snapshot;
+  try {
+    snapshot = JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch (err) {
+    logger.warn({ err, pid }, "incident-replay: corrupt snapshot file");
+    return null;
+  }
+  let replayedTag = null;
+  let replayedScore = null;
+  let replayedHint = null;
+  try {
+    const causal = await buildCausalChain(
+      snapshot.logs,
+      snapshot.network,
+      snapshot.contexts,
+      snapshot.firedAt,
+      snapshot.terminal,
+      snapshot.processExits,
+      snapshot.ciEvents,
+      snapshot.deployments,
+      snapshot.infraEvents
+    );
+    const top = causal.hypotheses[0] ?? null;
+    replayedTag = top?.tag ?? null;
+    replayedScore = top?.confidenceScore ?? null;
+    replayedHint = top?.fixHint ?? null;
+  } catch (err) {
+    logger.warn({ err, pid }, "incident-replay: replay analysis threw \u2014 returning partial result");
+  }
+  const topTagChanged = replayedTag !== snapshot.originalTag;
+  const fixHintChanged = replayedHint !== snapshot.originalFixHint;
+  const confidenceDelta = replayedScore !== null && snapshot.originalConfidenceScore !== null ? Math.round((replayedScore - snapshot.originalConfidenceScore) * 1e3) / 1e3 : null;
+  const parts = [];
+  if (topTagChanged) {
+    parts.push(`Diagnosis changed: ${snapshot.originalTag ?? "none"} \u2192 ${replayedTag ?? "none"}`);
+  } else {
+    parts.push("Same diagnosis");
+  }
+  if (confidenceDelta !== null && Math.abs(confidenceDelta) >= 0.01) {
+    parts.push(`Confidence ${confidenceDelta >= 0 ? "+" : ""}${(confidenceDelta * 100).toFixed(1)}pp`);
+  }
+  if (fixHintChanged) parts.push("Fix hint changed");
+  return {
+    pid,
+    originalHypothesis: {
+      tag: snapshot.originalTag,
+      confidenceScore: snapshot.originalConfidenceScore,
+      fixHint: snapshot.originalFixHint
+    },
+    replayedHypothesis: {
+      tag: replayedTag,
+      confidenceScore: replayedScore,
+      fixHint: replayedHint
+    },
+    drift: {
+      topTagChanged,
+      confidenceDelta,
+      fixHintChanged,
+      summary: parts.join(". ") + "."
+    },
+    replayedAt: (/* @__PURE__ */ new Date()).toISOString()
+  };
+}
+async function replayAllSnapshots(opts = {}) {
+  const confidenceThreshold = opts.confidenceThreshold ?? 0.05;
+  let pids = listSnapshotPids();
+  if (opts.excludePrefix) pids = pids.filter((p) => !p.startsWith(opts.excludePrefix));
+  const total = pids.length;
+  if (opts.limit !== void 0) pids = pids.slice(0, Math.max(0, opts.limit));
+  let replayed = 0, skipped = 0, unchanged = 0, tagChanged = 0, fixHintChanged = 0, confidenceDrifted = 0;
+  const drifted = [];
+  for (const pid of pids) {
+    const result = await replayIncident(pid);
+    if (!result) {
+      skipped++;
+      continue;
+    }
+    replayed++;
+    const confDrift = result.drift.confidenceDelta !== null && Math.abs(result.drift.confidenceDelta) >= confidenceThreshold;
+    if (result.drift.topTagChanged) tagChanged++;
+    if (result.drift.fixHintChanged) fixHintChanged++;
+    if (confDrift) confidenceDrifted++;
+    if (result.drift.topTagChanged || result.drift.fixHintChanged || confDrift) {
+      drifted.push({
+        pid,
+        summary: result.drift.summary,
+        originalTag: result.originalHypothesis.tag,
+        replayedTag: result.replayedHypothesis.tag,
+        confidenceDelta: result.drift.confidenceDelta
+      });
+    } else {
+      unchanged++;
+    }
+  }
+  drifted.sort((a, b) => Number(b.originalTag !== b.replayedTag) - Number(a.originalTag !== a.replayedTag));
+  return {
+    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    confidenceThreshold,
+    total,
+    replayed,
+    skipped,
+    unchanged,
+    tagChanged,
+    fixHintChanged,
+    confidenceDrifted,
+    driftRate: replayed > 0 ? Math.round((replayed - unchanged) / replayed * 1e3) / 1e3 : 0,
+    drifted
+  };
+}
+function listSnapshotPids() {
+  try {
+    if (!fs.existsSync(REPLAY_DIR)) return [];
+    return fs.readdirSync(REPLAY_DIR).filter((f) => f.endsWith(".json")).map((f) => f.slice(0, -5)).sort();
+  } catch {
+    return [];
+  }
+}
+export {
+  captureSnapshot,
+  listSnapshotPids,
+  replayAllSnapshots,
+  replayIncident
+};

package/dist/intelligence/incident-result-cache.js

@@ -0,0 +1,33 @@
+const TTL_MS = 60 * 60 * 1e3;
+const MAX_ENTRIES = 500;
+const _cache = /* @__PURE__ */ new Map();
+function cacheIncidentResult(result) {
+  _cache.set(result.fingerprint, { ...result, cachedAt: Date.now() });
+  if (_cache.size > MAX_ENTRIES) {
+    const cutoff = Date.now() - TTL_MS;
+    for (const [k, v] of _cache) {
+      if (v.cachedAt < cutoff) _cache.delete(k);
+    }
+  }
+}
+function getCachedIncidentResult(fingerprint) {
+  const entry = _cache.get(fingerprint);
+  if (!entry) return null;
+  if (Date.now() - entry.cachedAt > TTL_MS) {
+    _cache.delete(fingerprint);
+    return null;
+  }
+  return entry;
+}
+function invalidateIncidentResult(fingerprint) {
+  _cache.delete(fingerprint);
+}
+function incidentCacheSize() {
+  return _cache.size;
+}
+export {
+  cacheIncidentResult,
+  getCachedIncidentResult,
+  incidentCacheSize,
+  invalidateIncidentResult
+};

package/dist/intelligence/incident-similarity.js

@@ -0,0 +1,58 @@
+import { incidentStore } from "../sensor/incident-store.js";
+import { memoryStore } from "../datadog/memory-store.js";
+import { hybridSearch } from "./postmortem-retrieval.js";
+const DAY_MS = 864e5;
+function findSimilarIncidents(ctx, limit = 3) {
+  const results = [];
+  const now = Date.now();
+  if (ctx.service) {
+    try {
+      const serviceLower = ctx.service.toLowerCase();
+      for (const inc of incidentStore.list(void 0, 100)) {
+        const match = (inc.service ?? "").toLowerCase() === serviceLower || (inc.hypothesis ?? "").toLowerCase().includes(serviceLower);
+        if (match) {
+          results.push({
+            hypothesis: inc.hypothesis,
+            daysAgo: Math.floor((now - inc.createdAt) / DAY_MS),
+            confidence: inc.confidence,
+            source: "service_history"
+          });
+        }
+      }
+    } catch {
+    }
+  }
+  for (const file of (ctx.files ?? []).slice(0, 10)) {
+    try {
+      for (const mem of memoryStore.findByFile(file, 2)) {
+        results.push({
+          hypothesis: `${file}: ${mem.pdAlertTitle || mem.errorType || mem.errorMessage}`,
+          daysAgo: Math.floor((now - mem.firedAt) / DAY_MS),
+          confidence: 0.7,
+          // memory-store hits are exact filename matches — treated as high-confidence
+          source: "file_history"
+        });
+      }
+    } catch {
+    }
+  }
+  if (ctx.commandText && ctx.commandText.trim().length >= 3) {
+    try {
+      const hits = hybridSearch(ctx.commandText, { topK: 2, maxCorpus: 200 });
+      for (const hit of hits) {
+        results.push({
+          hypothesis: hit.postmortem.rootCause,
+          daysAgo: 0,
+          confidence: hit.postmortem.confidence,
+          source: "text_match"
+        });
+      }
+    } catch {
+    }
+  }
+  results.sort((a, b) => b.confidence - a.confidence || a.daysAgo - b.daysAgo);
+  return results.slice(0, limit);
+}
+export {
+  findSimilarIncidents
+};

package/dist/intelligence/infra-detectors.js

@@ -0,0 +1,312 @@
+import { scoreToConfidence } from "./detectors.js";
+const DATADOG_BOOST = 0.15;
+function withDatadogBoost(base, matched) {
+  return matched.some((e) => e.source === "datadog") ? Math.min(base + DATADOG_BOOST, 0.95) : base;
+}
+function detectDbConnectionPool(events) {
+  const matches = events.filter((e) => e.kind === "db_connection_pool_exhausted");
+  if (matches.length === 0) return null;
+  const p = matches[0];
+  const score = withDatadogBoost(0.8, matches);
+  return {
+    tag: "infra_db_connection_pool",
+    summary: `Database connection pool exhausted on \`${p.service}\` \u2014 ${p.message}`,
+    confidence: scoreToConfidence(score),
+    confidenceScore: score,
+    // Fix reliability is lower than diagnostic confidence: resize vs restart vs leak hunt
+    // depends on the specific failure mode and cannot be determined from telemetry alone.
+    remediationConfidence: 0.6,
+    evidence: [
+      `Triggering signal: "${p.message.slice(0, 200)}"`,
+      `Service: \`${p.service}\``,
+      `Endpoint: \`${p.attributes.endpoint || "unknown"}\``,
+      `Trace: \`${p.attributes.traceId || "none"}\``
+    ],
+    causalPath: [
+      "Request volume exceeded connection pool capacity",
+      "New connections queued waiting for a free slot",
+      `\`${p.service}\` \u2192 timeout / connection refused on \`${p.attributes.endpoint || "DB"}\``
+    ],
+    fixHint: [
+      "Increase pool size: set `DB_POOL_MAX` (or `pool.max` in your ORM config).",
+      "Check for connection leaks \u2014 unclosed transactions hold pool slots indefinitely.",
+      "Add a connection pool metrics query to validate utilisation."
+    ].join(" "),
+    fixAction: { type: "service_restart", target: p.service, method: "kubectl" }
+  };
+}
+function detectOomKill(events) {
+  const matches = events.filter((e) => e.kind === "oom_kill" || e.kind === "memory_pressure");
+  if (matches.length === 0) return null;
+  const p = matches[0];
+  const isHard = p.kind === "oom_kill";
+  const score = withDatadogBoost(isHard ? 0.9 : 0.65, matches);
+  const exitCode = p.attributes.exitCode;
+  const memMb = p.attributes.memoryLimitMb;
+  return {
+    tag: "infra_oom_kill",
+    summary: `${isHard ? "OOM kill" : "Memory pressure"} on \`${p.service}\` \u2014 ${p.message}`,
+    confidence: scoreToConfidence(score),
+    confidenceScore: score,
+    // Raising the memory limit restarts the service but does not fix the underlying leak.
+    // Remediation confidence is lower: the hint is directionally correct but rarely complete.
+    remediationConfidence: isHard ? 0.55 : 0.5,
+    evidence: [
+      `Triggering signal: "${p.message.slice(0, 200)}"`,
+      `Service: \`${p.service}\``,
+      ...isHard ? [`Exit code ${exitCode} (SIGKILL \u2014 kernel OOM killer fired)`] : [],
+      ...memMb ? [`Memory limit: ${memMb} MB`] : []
+    ],
+    causalPath: [
+      "RSS / heap grew past the container memory limit",
+      "Kernel OOM killer selected and killed the process (exit 137)",
+      isHard ? `\`${p.service}\` is now restarting \u2014 possible CrashLoopBackOff` : `\`${p.service}\` experiencing memory pressure \u2014 degraded performance`
+    ],
+    fixHint: isHard ? [
+      "Increase container memory limit in k8s manifest / docker-compose.",
+      "Heap-profile with `node --inspect` or `py-spy top` to find the leak.",
+      "Check for unbounded caches or retained buffers in recent deploys."
+    ].join(" ") : [
+      "Profile heap allocation. Check for unbounded in-memory caches or event-listener leaks.",
+      "Review recent deploys for large data structure growth."
+    ].join(" "),
+    fixAction: isHard ? { type: "service_restart", target: p.service, method: "kubectl" } : void 0
+  };
+}
+function detectRateLimitCascade(events) {
+  const matches = events.filter((e) => e.kind === "rate_limit_cascade");
+  if (matches.length === 0) return null;
+  const p = matches[0];
+  const score = withDatadogBoost(0.75, matches);
+  return {
+    tag: "infra_rate_limit_cascade",
+    summary: `Rate-limit cascade on \`${p.service}\` \u2014 ${p.message}`,
+    confidence: scoreToConfidence(score),
+    confidenceScore: score,
+    // The fix (add backoff + honour Retry-After) is correct but requires a code change
+    // and deploy — not an immediately executable command. Remediation confidence reflects
+    // that the hint cannot be auto-applied without human involvement.
+    remediationConfidence: 0.65,
+    evidence: [
+      `Triggering signal: "${p.message.slice(0, 200)}"`,
+      `Service: \`${p.service}\``,
+      `Endpoint: \`${p.attributes.endpoint || "unknown"}\``
+    ],
+    causalPath: [
+      "Upstream API / gateway returned 429 Too Many Requests",
+      "Service retried without exponential backoff, amplifying the traffic spike",
+      `\`${p.service}\` \u2192 retry storm \u2192 all requests throttled`
+    ],
+    fixHint: [
+      "Add exponential backoff with jitter to the retry loop.",
+      "Honour the `Retry-After` response header.",
+      "Consider a token-bucket circuit breaker to shed load before hitting the upstream limit."
+    ].join(" ")
+  };
+}
+function detectDownstreamLatency(events) {
+  const matches = events.filter(
+    (e) => e.kind === "downstream_latency_spike" || e.kind === "slow_query"
+  );
+  if (matches.length === 0) return null;
+  const p = matches[0];
+  const isQuery = p.kind === "slow_query";
+  const score = withDatadogBoost(isQuery ? 0.7 : 0.6, matches);
+  return {
+    tag: isQuery ? "infra_slow_query" : "infra_downstream_latency",
+    summary: `${isQuery ? "Slow DB query" : "Downstream latency spike"} on \`${p.service}\` \u2014 ${p.message}`,
+    confidence: scoreToConfidence(score),
+    confidenceScore: score,
+    evidence: [
+      `Triggering signal: "${p.message.slice(0, 200)}"`,
+      `Service: \`${p.service}\``,
+      `Endpoint: \`${p.attributes.endpoint || "unknown"}\``
+    ],
+    causalPath: isQuery ? [
+      "Query plan degraded (missing index, table scan, lock contention)",
+      `Database query exceeded statement_timeout on \`${p.service}\``,
+      "Upstream requests queued waiting \u2014 goroutines / workers exhausted"
+    ] : [
+      "Downstream service p99 latency exceeded caller timeout",
+      "Threads / goroutines blocked on slow dependency",
+      `\`${p.service}\` \u2192 cascading timeouts to callers`
+    ],
+    fixHint: isQuery ? "Run EXPLAIN ANALYZE on the slow query. Add indexes on WHERE / JOIN columns. Check recent schema migrations for unindexed columns." : "Add circuit breaker around the slow downstream. Increase timeout if the dependency is legitimately slow, or reduce p99 of the dependency."
+  };
+}
+function detectCertificateExpiry(events) {
+  const matches = events.filter((e) => e.kind === "certificate_expiry");
+  if (matches.length === 0) return null;
+  const p = matches[0];
+  const score = 0.85;
+  return {
+    tag: "infra_certificate_expiry",
+    summary: `TLS / certificate error on \`${p.service}\` \u2014 ${p.message}`,
+    confidence: scoreToConfidence(score),
+    confidenceScore: score,
+    // certbot renew is deterministic when the cert is genuinely expired — high remediation
+    // confidence. Mismatch cases (wrong hostname, CA trust) require manual intervention,
+    // but certificate expiry is the dominant failure mode by volume.
+    remediationConfidence: 0.9,
+    evidence: [
+      `Triggering signal: "${p.message.slice(0, 200)}"`,
+      `Service: \`${p.service}\``,
+      "Error pattern: TLS handshake failure / certificate validation error"
+    ],
+    causalPath: [
+      "TLS handshake rejected \u2014 certificate expired, untrusted, or hostname mismatch",
+      `\`${p.service}\` cannot establish HTTPS connection to endpoint`,
+      "All HTTPS requests to this endpoint failing"
+    ],
+    fixHint: [
+      "Check expiry: `openssl s_client -connect <host>:443 2>/dev/null | openssl x509 -noout -dates`",
+      "Renew via Let's Encrypt (`certbot renew`) or your CA.",
+      "Verify the cert covers the target hostname (SAN list)."
+    ].join(" ")
+  };
+}
+function detectDiskPressure(events) {
+  const matches = events.filter((e) => e.kind === "disk_pressure");
+  if (matches.length === 0) return null;
+  const p = matches[0];
+  const score = 0.85;
+  return {
+    tag: "infra_disk_pressure",
+    summary: `Disk pressure on \`${p.service}\` \u2014 ${p.message}`,
+    confidence: scoreToConfidence(score),
+    confidenceScore: score,
+    evidence: [
+      `Triggering signal: "${p.message.slice(0, 200)}"`,
+      `Service: \`${p.service}\``,
+      "Error pattern: no space left / disk full"
+    ],
+    causalPath: [
+      "Filesystem reached 100% capacity",
+      "Writes blocked \u2014 log rotation, WAL, or temp files cannot be created",
+      `\`${p.service}\` \u2192 I/O errors / service crash`
+    ],
+    fixHint: [
+      "Check disk usage: `df -h` and `du -sh /var/log/* /tmp/*`.",
+      "Rotate logs immediately: `journalctl --vacuum-size=500M`.",
+      "Increase PV size or add log retention policy to prevent recurrence."
+    ].join(" ")
+  };
+}
+function detectQueueBacklog(events) {
+  const matches = events.filter((e) => e.kind === "queue_backlog");
+  if (matches.length === 0) return null;
+  const p = matches[0];
+  const score = withDatadogBoost(0.65, matches);
+  return {
+    tag: "infra_queue_backlog",
+    summary: `Message queue backlog on \`${p.service}\` \u2014 ${p.message}`,
+    confidence: scoreToConfidence(score),
+    confidenceScore: score,
+    // Scaling consumers is directionally correct but may not be the fastest path —
+    // slow handler logic or a poison message may require different intervention.
+    remediationConfidence: 0.6,
+    evidence: [
+      `Triggering signal: "${p.message.slice(0, 200)}"`,
+      `Service: \`${p.service}\``,
+      "Error pattern: consumer lag / queue depth / backlog"
+    ],
+    causalPath: [
+      "Producer rate exceeded consumer throughput",
+      "Consumer group lagging \u2014 messages accumulating in queue",
+      `\`${p.service}\` \u2192 processing delay / stale data`
+    ],
+    fixHint: [
+      "Scale up consumers (increase replica count or partition count).",
+      "Check for slow message processing \u2014 add timing metrics around handler logic.",
+      "Verify no consumer is in an error loop silently dropping messages."
+    ].join(" ")
+  };
+}
+function detectServiceUnavailable(events) {
+  const matches = events.filter((e) => e.kind === "service_unavailable");
+  if (matches.length === 0) return null;
+  const p = matches[0];
+  const score = withDatadogBoost(0.7, matches);
+  return {
+    tag: "infra_service_unavailable",
+    summary: `Service unavailable: \`${p.service}\` cannot reach upstream \u2014 ${p.message}`,
+    confidence: scoreToConfidence(score),
+    confidenceScore: score,
+    evidence: [
+      `Triggering signal: "${p.message.slice(0, 200)}"`,
+      `Service: \`${p.service}\``,
+      `Endpoint: \`${p.attributes.endpoint || "unknown"}\``,
+      "Error pattern: 503 / upstream connect error / no healthy upstream"
+    ],
+    causalPath: [
+      "Upstream service is down or all instances are unhealthy",
+      "Load balancer / ingress cannot find a healthy backend",
+      `\`${p.service}\` receiving 503 on every request to the dependency`
+    ],
+    fixHint: [
+      "Check k8s pod status: `kubectl get pods -n <namespace>`.",
+      "Look for CrashLoopBackOff or Pending pods.",
+      "Review readiness probe failures and recent deploy events."
+    ].join(" ")
+  };
+}
+const NOISE_ENDPOINT_PATTERNS = [
+  "/health",
+  "/metrics",
+  "/ready",
+  "/liveness",
+  "/readiness",
+  "/ping",
+  "/livez",
+  "/readyz"
+];
+function isMonitoringNoise(e) {
+  const endpoint = String(e.attributes.endpoint || "");
+  const msg = e.message || "";
+  return NOISE_ENDPOINT_PATTERNS.some((p) => endpoint.includes(p) || msg.includes(p));
+}
+function detectUpstreamError(events) {
+  const matches = events.filter((e) => e.kind === "upstream_error" && !isMonitoringNoise(e));
+  if (matches.length === 0) return null;
+  const p = matches[0];
+  const score = 0.4;
+  return {
+    tag: "infra_upstream_error",
+    summary: `Production error on \`${p.service}\` \u2014 ${p.message}`,
+    confidence: scoreToConfidence(score),
+    confidenceScore: score,
+    evidence: [
+      `Service: \`${p.service}\``,
+      `Endpoint: \`${p.attributes.endpoint || "unknown"}\``,
+      `Trace: \`${p.attributes.traceId || "none"}\``
+    ],
+    causalPath: [
+      `\`${p.service}\` returned an error on \`${p.attributes.endpoint || "unknown"}\``
+    ],
+    fixHint: p.attributes.traceId ? `Inspect trace \`${p.attributes.traceId}\` in Datadog APM for the full call stack. Check service logs around the incident time.` : "Check service logs and recent deploys. Run `get_incident_context` to fetch the full Datadog trace."
+  };
+}
+const ALL_INFRA_DETECTORS = [
+  detectDbConnectionPool,
+  detectOomKill,
+  detectRateLimitCascade,
+  detectDownstreamLatency,
+  detectCertificateExpiry,
+  detectDiskPressure,
+  detectQueueBacklog,
+  detectServiceUnavailable,
+  detectUpstreamError
+  // catch-all — must be last
+];
+export {
+  ALL_INFRA_DETECTORS,
+  detectCertificateExpiry,
+  detectDbConnectionPool,
+  detectDiskPressure,
+  detectDownstreamLatency,
+  detectOomKill,
+  detectQueueBacklog,
+  detectRateLimitCascade,
+  detectServiceUnavailable,
+  detectUpstreamError
+};