mergen-server 1.4.0

package/dist/intelligence/policy-sync.js

@@ -0,0 +1,103 @@
+import crypto from "crypto";
+import { loadEnterprisePolicy, saveEnterprisePolicy, IMMUTABLE_RULE_IDS } from "./enterprise-policy-engine.js";
+import logger from "../sensor/logger.js";
+let _lastEtag = "";
+function verifyPolicySignature(body, signature, secret) {
+  if (!signature) return false;
+  const [algo, hex] = signature.split("=");
+  if (algo !== "sha256" || !hex) return false;
+  const expected = crypto.createHmac("sha256", secret).update(body).digest("hex");
+  return crypto.timingSafeEqual(Buffer.from(hex, "hex"), Buffer.from(expected, "hex"));
+}
+async function fetchAndApply(opts) {
+  const hmacSecret = process.env.MERGEN_POLICY_HMAC_SECRET;
+  const headers = { "Accept": "application/json" };
+  if (_lastEtag) headers["If-None-Match"] = _lastEtag;
+  let resp;
+  try {
+    resp = await fetch(opts.url, { headers, signal: AbortSignal.timeout(1e4) });
+  } catch (err) {
+    logger.warn({ err, url: opts.url }, "policy-sync: fetch failed (non-fatal)");
+    return;
+  }
+  if (resp.status === 304) {
+    logger.debug({ url: opts.url }, "policy-sync: policy unchanged");
+    return;
+  }
+  if (!resp.ok) {
+    logger.warn({ status: resp.status, url: opts.url }, "policy-sync: remote returned non-2xx (non-fatal)");
+    return;
+  }
+  const etag = resp.headers.get("etag") ?? "";
+  let rawBody;
+  try {
+    rawBody = await resp.text();
+  } catch (err) {
+    logger.warn({ err }, "policy-sync: could not read response body");
+    return;
+  }
+  if (hmacSecret) {
+    const sig = resp.headers.get("x-mergen-policy-signature");
+    if (!verifyPolicySignature(rawBody, sig, hmacSecret)) {
+      logger.error(
+        { url: opts.url, hasSignature: !!sig },
+        "policy-sync: HMAC signature verification failed \u2014 remote policy rejected. Ensure the policy server sends X-Mergen-Policy-Signature: sha256=<hmac> keyed with MERGEN_POLICY_HMAC_SECRET."
+      );
+      return;
+    }
+    logger.debug({ url: opts.url }, "policy-sync: HMAC signature verified");
+  }
+  let incoming;
+  try {
+    incoming = JSON.parse(rawBody);
+  } catch (err) {
+    logger.warn({ err }, "policy-sync: could not parse remote policy JSON");
+    return;
+  }
+  const local = loadEnterprisePolicy();
+  const mode = opts.mergeMode ?? "replace";
+  let merged;
+  if (mode === "merge") {
+    const existingIds = new Set(local.rules.map((r) => r.id));
+    merged = { ...local, rules: [...local.rules, ...incoming.rules.filter((r) => !existingIds.has(r.id))] };
+  } else {
+    const immutableRules = local.rules.filter((r) => IMMUTABLE_RULE_IDS.has(r.id));
+    const remoteRulesWithoutImmutable = incoming.rules.filter((r) => !IMMUTABLE_RULE_IDS.has(r.id));
+    merged = { ...incoming, rules: [...immutableRules, ...remoteRulesWithoutImmutable] };
+    const keptIds = immutableRules.map((r) => r.id);
+    if (keptIds.length > 0) {
+      logger.info({ keptIds }, "policy-sync: preserved immutable local rules during replace");
+    }
+  }
+  try {
+    saveEnterprisePolicy(merged);
+    _lastEtag = etag;
+    logger.info({ url: opts.url, mode, ruleCount: merged.rules.length }, "policy-sync: policy synced");
+  } catch (err) {
+    logger.warn({ err }, "policy-sync: failed to save synced policy (non-fatal)");
+  }
+}
+function startPolicySync(opts) {
+  const intervalMs = opts.intervalMs ?? 5 * 60 * 1e3;
+  if (!opts.url.startsWith("https://")) {
+    logger.error(
+      { url: opts.url },
+      "policy-sync: MERGEN_POLICY_URL must use HTTPS to prevent policy injection over plain HTTP. Update MERGEN_POLICY_URL to an https:// address and restart."
+    );
+    return;
+  }
+  const hmacSecret = process.env.MERGEN_POLICY_HMAC_SECRET;
+  if (!hmacSecret) {
+    logger.warn(
+      { url: opts.url },
+      "policy-sync: MERGEN_POLICY_HMAC_SECRET is not set \u2014 remote policy responses are not signature-verified. A compromised MERGEN_POLICY_URL could inject permissive rules. Set MERGEN_POLICY_HMAC_SECRET to a shared secret and configure the policy server to sign responses."
+    );
+  }
+  void fetchAndApply(opts);
+  const timer = setInterval(() => void fetchAndApply(opts), intervalMs);
+  timer.unref();
+  logger.info({ url: opts.url, intervalMs, mergeMode: opts.mergeMode ?? "replace", hmacVerification: !!hmacSecret }, "policy-sync: started");
+}
+export {
+  startPolicySync
+};

package/dist/intelligence/policy-test-runner.js

@@ -0,0 +1,35 @@
+import { evaluateEnterprisePolicy } from "./enterprise-policy-engine.js";
+function runPolicyTestCases(cases, policyOverride) {
+  const results = cases.map((tc) => {
+    const result = evaluateEnterprisePolicy(tc.input, policyOverride);
+    return {
+      id: tc.id,
+      description: tc.description,
+      expected: tc.expectedVerdict,
+      got: result.verdict,
+      passed: result.verdict === tc.expectedVerdict,
+      reasons: result.reasons
+    };
+  });
+  const passed = results.filter((r) => r.passed).length;
+  return { results, passed, failed: results.length - passed, total: results.length, allPassed: passed === results.length };
+}
+function parsePolicyTestFile(raw) {
+  const cases = Array.isArray(raw) ? raw : raw?.cases;
+  if (!Array.isArray(cases)) {
+    throw new Error('Expected a JSON array of test cases, or an object with a top-level "cases" array.');
+  }
+  for (const [i, tc] of cases.entries()) {
+    const c = tc;
+    if (typeof c?.id !== "string" || !c.id) throw new Error(`Test case at index ${i} is missing a string "id".`);
+    if (!c.input || typeof c.input !== "object") throw new Error(`Test case "${c.id}" is missing an "input" object.`);
+    if (!["block", "warn", "pass"].includes(c.expectedVerdict)) {
+      throw new Error(`Test case "${c.id}" has an invalid "expectedVerdict" (must be block/warn/pass).`);
+    }
+  }
+  return cases;
+}
+export {
+  parsePolicyTestFile,
+  runPolicyTestCases
+};

package/dist/intelligence/postmortem-parser.js

@@ -0,0 +1,181 @@
+import fs from "fs";
+import path from "path";
+import fg from "fast-glob";
+import { createHash } from "crypto";
+import { adrStore } from "../sensor/adr-store.js";
+import { postmortemStore } from "./postmortem-store.js";
+import logger from "../sensor/logger.js";
+function parseMttrToMs(text) {
+  if (!text || text === "unknown") return null;
+  const minMatch = text.match(/(\d+)m/i);
+  const secMatch = text.match(/(\d+)s/i);
+  let ms = 0;
+  if (minMatch) ms += parseInt(minMatch[1], 10) * 6e4;
+  if (secMatch) ms += parseInt(secMatch[1], 10) * 1e3;
+  if (ms > 0) return ms;
+  const num = parseInt(text, 10);
+  if (!isNaN(num)) return num * 1e3;
+  return null;
+}
+function deterministicUUID(input) {
+  const hash = createHash("md5").update(input).digest("hex");
+  return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
+}
+function parseAdrMarkdown(content) {
+  const titleMatch = content.match(/^#\s*(ADR-\d+):\s*(.*)/mi);
+  if (!titleMatch) return null;
+  const id = titleMatch[1].toUpperCase();
+  const title = titleMatch[2].trim();
+  const dateMatch = content.match(/\*\*Date:\*\*\s*(.*)/i);
+  const date = dateMatch ? dateMatch[1].trim() : (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+  const statusMatch = content.match(/\*\*Status:\*\*\s*(.*)/i);
+  let status = "proposed";
+  if (statusMatch) {
+    const s = statusMatch[1].trim().toLowerCase();
+    if (s === "accepted" || s === "deprecated" || s === "superseded" || s === "proposed") {
+      status = s;
+    }
+  }
+  const extractSection = (headerName) => {
+    const regex = new RegExp(`##\\s*${headerName}[\\s\\S]*?(?=##|$)`, "i");
+    const match = content.match(regex);
+    if (!match) return "";
+    return match[0].replace(new RegExp(`^##\\s*${headerName}`, "i"), "").trim();
+  };
+  const decision = extractSection("Decision");
+  const rationale = extractSection("Rationale");
+  const consequences = extractSection("Consequences");
+  const alternativesRaw = extractSection("Alternatives(?: considered)?");
+  const alternatives = [];
+  if (alternativesRaw) {
+    const items = alternativesRaw.match(/^\s*-\s*(.*)/gm);
+    if (items) {
+      for (const item of items) {
+        alternatives.push(item.replace(/^\s*-\s*/, "").trim());
+      }
+    } else {
+      alternatives.push(alternativesRaw);
+    }
+  }
+  return {
+    id,
+    title,
+    status,
+    date,
+    decision,
+    alternatives,
+    rationale,
+    consequences
+  };
+}
+function parsePostmortemMarkdown(filename, content) {
+  const tagMatch = content.match(/^#\s*Postmortem\s*—\s*(.*)/mi);
+  if (!tagMatch) return null;
+  let tag = tagMatch[1].trim();
+  if (!tag.startsWith("infra_")) {
+    tag = `infra_${tag}`;
+  }
+  const serviceMatch = content.match(/\*\*Service:\*\*\s*([^\s|]+)/i);
+  const service = serviceMatch ? serviceMatch[1].trim() : "unknown";
+  const dateMatch = content.match(/\*\*Date:\*\*\s*([^\s|]+)/i);
+  const dateStr = dateMatch ? dateMatch[1].trim() : "";
+  const generatedAt = dateStr ? new Date(dateStr).getTime() : Date.now();
+  const confidenceMatch = content.match(/\*\*Confidence:\*\*\s*(\d+)%/i);
+  const confidence = confidenceMatch ? parseInt(confidenceMatch[1], 10) / 100 : 0.85;
+  const mttrMatch = content.match(/\*\*MTTR:\*\*\s*([^|]+)/i);
+  const mttrMs = mttrMatch ? parseMttrToMs(mttrMatch[1].trim()) : null;
+  const resolutionMatch = content.match(/\*\*Resolution:\*\*\s*(.*)/i);
+  const resolvedAutonomously = resolutionMatch ? /autonomous|mergen/i.test(resolutionMatch[1]) : false;
+  const branchMatch = content.match(/\*\*Branch:\*\*\s*([^\s|]+)/i);
+  const gitBranch = branchMatch ? branchMatch[1].trim() : null;
+  const shaMatch = content.match(/\*\*SHA:\*\*\s*([^\s|]+)/i);
+  const gitSha = shaMatch ? shaMatch[1].trim() : null;
+  const extractSection = (headerName) => {
+    const regex = new RegExp(`##\\s*${headerName}[\\s\\S]*?(?=##|$)`, "i");
+    const match = content.match(regex);
+    if (!match) return "";
+    return match[0].replace(new RegExp(`^##\\s*${headerName}`, "i"), "").trim();
+  };
+  const rootCause = extractSection("Root Cause") || "Unknown root cause";
+  const fixAppliedRaw = extractSection("Fix Applied");
+  let fixCommand = null;
+  if (fixAppliedRaw) {
+    const codeFenceMatch = fixAppliedRaw.match(/```(?:[a-zA-Z]*)\n([\s\S]*?)\n```/);
+    fixCommand = codeFenceMatch ? codeFenceMatch[1].trim() : fixAppliedRaw.trim();
+  }
+  const pid = deterministicUUID(path.basename(filename));
+  return {
+    pid,
+    tag,
+    service,
+    gitSha,
+    gitBranch,
+    rootCause,
+    fixCommand,
+    confidence,
+    mttrMs,
+    resolvedAutonomously,
+    causallyCorrect: true,
+    // imported postmortem was verified correct
+    generatedAt,
+    body: content
+  };
+}
+function findRepoRoot(startPath = process.cwd()) {
+  let curr = startPath;
+  while (curr && curr !== path.parse(curr).root) {
+    if (fs.existsSync(path.join(curr, ".git")) || fs.existsSync(path.join(curr, "docs/adr"))) {
+      return curr;
+    }
+    curr = path.dirname(curr);
+  }
+  return startPath;
+}
+async function syncMarkdownFilesFromDisk(startPath = process.cwd()) {
+  let adrsSynced = 0;
+  let postmortemsSynced = 0;
+  const repoRoot = findRepoRoot(startPath);
+  try {
+    const adrPattern = path.join(repoRoot, "docs/adr/ADR-*.md").replace(/\\/g, "/");
+    const adrFiles = await fg(adrPattern);
+    for (const file of adrFiles) {
+      try {
+        const content = fs.readFileSync(file, "utf8");
+        const adr = parseAdrMarkdown(content);
+        if (adr) {
+          adrStore.upsert(adr);
+          adrsSynced++;
+        }
+      } catch (err) {
+        logger.warn({ file, err }, "postmortem-parser: failed to sync ADR file");
+      }
+    }
+    const postmortemDir = path.join(repoRoot, "docs/postmortems");
+    if (fs.existsSync(postmortemDir)) {
+      const pmPattern = path.join(postmortemDir, "*.md").replace(/\\/g, "/");
+      const pmFiles = await fg(pmPattern);
+      for (const file of pmFiles) {
+        try {
+          const content = fs.readFileSync(file, "utf8");
+          const pm = parsePostmortemMarkdown(file, content);
+          if (pm) {
+            postmortemStore.write(pm);
+            postmortemsSynced++;
+          }
+        } catch (err) {
+          logger.warn({ file, err }, "postmortem-parser: failed to sync postmortem file");
+        }
+      }
+    }
+    logger.info({ adrsSynced, postmortemsSynced, repoRoot }, "postmortem-parser: sync completed");
+  } catch (err) {
+    logger.warn({ err }, "postmortem-parser: markdown files sync failed");
+  }
+  return { adrsSynced, postmortemsSynced };
+}
+export {
+  findRepoRoot,
+  parseAdrMarkdown,
+  parsePostmortemMarkdown,
+  syncMarkdownFilesFromDisk
+};

package/dist/intelligence/postmortem-retrieval.js

@@ -0,0 +1,224 @@
+import { postmortemStore } from "./postmortem-store.js";
+const RRF_K = 60;
+const STOPWORDS = /* @__PURE__ */ new Set([
+  "the",
+  "a",
+  "an",
+  "and",
+  "or",
+  "but",
+  "in",
+  "on",
+  "at",
+  "to",
+  "for",
+  "of",
+  "with",
+  "by",
+  "from",
+  "up",
+  "about",
+  "into",
+  "through",
+  "during",
+  "is",
+  "are",
+  "was",
+  "were",
+  "be",
+  "been",
+  "being",
+  "have",
+  "has",
+  "had",
+  "do",
+  "does",
+  "did",
+  "will",
+  "would",
+  "could",
+  "should",
+  "may",
+  "might",
+  "it",
+  "its",
+  "this",
+  "that",
+  "these",
+  "those",
+  "which",
+  "who",
+  "what",
+  "when",
+  "where",
+  "how",
+  "why",
+  "all",
+  "each",
+  "every",
+  "both",
+  "few",
+  "more",
+  "most",
+  "other",
+  "some",
+  "such",
+  "no",
+  "not",
+  "only",
+  "same",
+  "than",
+  "too",
+  "very",
+  "just",
+  "can",
+  "then",
+  "now",
+  "also",
+  "after",
+  "before",
+  "if",
+  "while",
+  "because",
+  "as",
+  "until",
+  "since",
+  "so"
+]);
+function tokenize(text) {
+  return text.toLowerCase().replace(/[^\w\s_-]/g, " ").replace(/[_-]+/g, " ").split(/\s+/).filter((t) => t.length >= 2 && !STOPWORDS.has(t));
+}
+function buildIdf(documents) {
+  const N = documents.length;
+  const df = /* @__PURE__ */ new Map();
+  for (const tokens of documents) {
+    const seen = new Set(tokens);
+    for (const t of seen) {
+      df.set(t, (df.get(t) ?? 0) + 1);
+    }
+  }
+  const idf = /* @__PURE__ */ new Map();
+  for (const [term, count] of df) {
+    idf.set(term, Math.log(1 + N / (1 + count)));
+  }
+  return idf;
+}
+function computeTfIdf(tokens, idf) {
+  const tf = /* @__PURE__ */ new Map();
+  for (const t of tokens) {
+    tf.set(t, (tf.get(t) ?? 0) + 1);
+  }
+  const vec = /* @__PURE__ */ new Map();
+  const total = tokens.length || 1;
+  for (const [t, count] of tf) {
+    const idfScore = idf.get(t) ?? Math.log(2);
+    vec.set(t, count / total * idfScore);
+  }
+  return vec;
+}
+function cosineSimilarity(a, b) {
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (const [t, wa] of a) {
+    const wb = b.get(t) ?? 0;
+    dot += wa * wb;
+    normA += wa * wa;
+  }
+  for (const [, wb] of b) {
+    normB += wb * wb;
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}
+function embeddingSearch(query, postmortems, topK) {
+  if (postmortems.length === 0) return [];
+  const docTexts = postmortems.map(
+    (pm) => [pm.tag, pm.service, pm.rootCause, pm.fixCommand ?? "", pm.body].join(" ")
+  );
+  const docTokens = docTexts.map(tokenize);
+  const queryTokens = tokenize(query);
+  if (queryTokens.length === 0) return [];
+  const idf = buildIdf([...docTokens, queryTokens]);
+  const queryVec = computeTfIdf(queryTokens, idf);
+  const scored = postmortems.map((pm, i) => ({
+    pid: pm.pid,
+    similarity: cosineSimilarity(queryVec, computeTfIdf(docTokens[i], idf))
+  }));
+  return scored.filter((s) => s.similarity > 0).sort((a, b) => b.similarity - a.similarity).slice(0, topK).map((s, rank) => ({ ...s, rank }));
+}
+function reciprocalRankFusion(lists, k = RRF_K) {
+  const scores = /* @__PURE__ */ new Map();
+  for (const list of lists) {
+    for (const { pid, rank } of list) {
+      scores.set(pid, (scores.get(pid) ?? 0) + 1 / (k + rank + 1));
+    }
+  }
+  return scores;
+}
+function hybridSearch(query, opts = {}) {
+  const { tag, service, topK = 10, maxCorpus = 500 } = opts;
+  if (!query.trim()) return [];
+  let corpus;
+  if (tag) {
+    corpus = postmortemStore.getByTag(tag, maxCorpus);
+  } else {
+    corpus = postmortemStore.list(maxCorpus);
+  }
+  if (service) {
+    corpus = corpus.filter((pm) => pm.service === service);
+  }
+  if (corpus.length === 0) return [];
+  const keywordRanking = postmortemStore.keywordSearch(query, topK * 3);
+  const corpusPidSet = new Set(corpus.map((pm) => pm.pid));
+  const filteredKeyword = keywordRanking.filter((r) => corpusPidSet.has(r.pid));
+  const embeddingRanking = embeddingSearch(query, corpus, topK * 3);
+  const rrfScores = reciprocalRankFusion([filteredKeyword, embeddingRanking]);
+  const keywordRankMap = new Map(filteredKeyword.map((r) => [r.pid, r.rank]));
+  const embeddingMap = new Map(embeddingRanking.map((r) => [r.pid, r]));
+  const allPids = /* @__PURE__ */ new Set([
+    ...filteredKeyword.map((r) => r.pid),
+    ...embeddingRanking.map((r) => r.pid)
+  ]);
+  const pmMap = new Map(corpus.map((pm) => [pm.pid, pm]));
+  const results = [];
+  for (const pid of allPids) {
+    const pm = pmMap.get(pid);
+    if (!pm) continue;
+    const score = rrfScores.get(pid) ?? 0;
+    const inKeyword = keywordRankMap.has(pid);
+    const inEmbedding = embeddingMap.has(pid);
+    results.push({
+      postmortem: pm,
+      score,
+      signals: [
+        ...inKeyword ? ["keyword"] : [],
+        ...inEmbedding ? ["embedding"] : []
+      ],
+      embeddingSimilarity: embeddingMap.get(pid)?.similarity ?? null,
+      keywordRank: keywordRankMap.get(pid) ?? null
+    });
+  }
+  return results.sort((a, b) => b.score - a.score).slice(0, topK);
+}
+function tagToQuery(tag) {
+  const expansions = {
+    db_connection_pool_exhausted: "database connection pool exhausted timeout max connections",
+    oom_kill: "out of memory OOM kill process killed memory pressure container",
+    memory_pressure: "memory pressure high usage RSS heap limit",
+    rate_limit_cascade: "rate limit 429 throttle cascade downstream retry storm",
+    slow_query: "slow query timeout database latency lock wait",
+    downstream_latency_spike: "downstream dependency latency spike timeout 503",
+    certificate_expiry: "certificate expiry TLS SSL expired renewal",
+    disk_pressure: "disk full pressure inode storage volume mount",
+    queue_backlog: "queue backlog lag consumer worker stuck processing",
+    service_unavailable: "service unavailable 503 health check crash restart"
+  };
+  const clean = tag.replace(/^infra_/, "");
+  return expansions[clean] ?? clean.replace(/_/g, " ");
+}
+export {
+  hybridSearch,
+  tagToQuery,
+  tokenize
+};