mergen-server 1.4.0

package/dist/sensor/policy-history.js

@@ -0,0 +1,147 @@
+import fs from "fs";
+import path from "path";
+import { randomUUID, createHash } from "crypto";
+import { DATA_DIR, zeroRetentionMode } from "./paths.js";
+import logger from "./logger.js";
+const POLICY_HISTORY_FILE = path.join(DATA_DIR, "policy-history.json");
+const POLICY_VERSIONS_DIR = path.join(DATA_DIR, "policy-versions");
+const MAX_HISTORY = 500;
+const MAX_VERSIONS = 500;
+let _entries = [];
+let _loaded = false;
+function _load() {
+  if (_loaded) return;
+  _loaded = true;
+  if (!fs.existsSync(POLICY_HISTORY_FILE)) return;
+  try {
+    const raw = JSON.parse(fs.readFileSync(POLICY_HISTORY_FILE, "utf8"));
+    if (raw?.version === 1 && Array.isArray(raw.entries)) {
+      _entries = raw.entries.slice(-MAX_HISTORY);
+    }
+  } catch (err) {
+    logger.warn({ err }, "policy-history: failed to load \u2014 starting fresh");
+  }
+}
+function _persist() {
+  if (zeroRetentionMode()) return;
+  try {
+    fs.mkdirSync(DATA_DIR, { recursive: true });
+    const tmp = `${POLICY_HISTORY_FILE}.tmp.${process.pid}`;
+    fs.writeFileSync(tmp, JSON.stringify({ version: 1, entries: _entries }), "utf8");
+    fs.renameSync(tmp, POLICY_HISTORY_FILE);
+  } catch (err) {
+    logger.warn({ err }, "policy-history: persist failed");
+  }
+}
+function _summariseRule(r) {
+  return { id: r.id, name: r.name, action: r.action };
+}
+function _versionIdFor(config) {
+  const hash = createHash("sha256").update(JSON.stringify(config)).digest("hex").slice(0, 16);
+  return `${Date.now()}-${hash}`;
+}
+function _versionPath(versionId) {
+  const safe = versionId.replace(/[^a-zA-Z0-9-]/g, "");
+  return path.join(POLICY_VERSIONS_DIR, `${safe}.json`);
+}
+function _writeSnapshot(config) {
+  if (zeroRetentionMode()) return null;
+  try {
+    fs.mkdirSync(POLICY_VERSIONS_DIR, { recursive: true });
+    const versionId = _versionIdFor(config);
+    fs.writeFileSync(_versionPath(versionId), JSON.stringify(config, null, 2), "utf8");
+    _pruneOldSnapshots();
+    return versionId;
+  } catch (err) {
+    logger.warn({ err }, "policy-history: failed to write policy version snapshot");
+    return null;
+  }
+}
+function _pruneOldSnapshots() {
+  try {
+    const files = fs.readdirSync(POLICY_VERSIONS_DIR).filter((f) => f.endsWith(".json")).sort();
+    const excess = files.length - MAX_VERSIONS;
+    if (excess > 0) {
+      for (const f of files.slice(0, excess)) {
+        try {
+          fs.unlinkSync(path.join(POLICY_VERSIONS_DIR, f));
+        } catch {
+        }
+      }
+    }
+  } catch {
+  }
+}
+function listPolicyVersions() {
+  try {
+    return fs.readdirSync(POLICY_VERSIONS_DIR).filter((f) => f.endsWith(".json")).map((f) => f.slice(0, -".json".length)).sort().reverse();
+  } catch {
+    return [];
+  }
+}
+function getPolicyVersion(versionId) {
+  try {
+    const raw = fs.readFileSync(_versionPath(versionId), "utf8");
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+function recordPolicyChange(before, after, actor = "unknown") {
+  _load();
+  const beforeRules = before?.rules ?? [];
+  const afterRules = after.rules;
+  const beforeIds = new Map(beforeRules.map((r) => [r.id, r]));
+  const afterIds = new Map(afterRules.map((r) => [r.id, r]));
+  const added = [];
+  const removed = [];
+  const modified = [];
+  for (const [id, rule] of afterIds) {
+    if (!beforeIds.has(id)) {
+      added.push(_summariseRule(rule));
+    } else {
+      const old = beforeIds.get(id);
+      const oldStr = JSON.stringify({ action: old.action, reason: old.reason, conditions: old.conditions });
+      const newStr = JSON.stringify({ action: rule.action, reason: rule.reason, conditions: rule.conditions });
+      if (oldStr !== newStr) {
+        modified.push({ id: rule.id, name: rule.name, from: oldStr.slice(0, 200), to: newStr.slice(0, 200) });
+      }
+    }
+  }
+  for (const [id, rule] of beforeIds) {
+    if (!afterIds.has(id)) removed.push(_summariseRule(rule));
+  }
+  const enabledChanged = before ? before.enabled !== after.enabled : null;
+  const versionId = _writeSnapshot(after);
+  const entry = {
+    id: randomUUID(),
+    changedAt: Date.now(),
+    actor,
+    rulesBefore: beforeRules.length,
+    rulesAfter: afterRules.length,
+    added,
+    removed,
+    modified,
+    enabledChanged,
+    enabledBefore: before?.enabled ?? null,
+    enabledAfter: after.enabled,
+    versionId
+  };
+  _entries.push(entry);
+  if (_entries.length > MAX_HISTORY) _entries = _entries.slice(-MAX_HISTORY);
+  _persist();
+  logger.info(
+    { added: added.length, removed: removed.length, modified: modified.length, actor },
+    "policy-history: change recorded"
+  );
+}
+function getPolicyHistory() {
+  _load();
+  return [..._entries].reverse();
+}
+export {
+  getPolicyHistory,
+  getPolicyVersion,
+  listPolicyVersions,
+  recordPolicyChange
+};

package/dist/sensor/pr-shadow-store.js

@@ -0,0 +1,83 @@
+import fs from "fs";
+import { randomUUID } from "crypto";
+import { PR_SHADOW_FILE, DATA_DIR } from "./paths.js";
+import logger from "./logger.js";
+const RING_SIZE = 500;
+let _results = [];
+let _loaded = false;
+function ensureLoaded() {
+  if (_loaded) return;
+  _loaded = true;
+  try {
+    const raw = fs.readFileSync(PR_SHADOW_FILE, "utf8");
+    const parsed = JSON.parse(raw);
+    if (parsed.version === 1 && Array.isArray(parsed.results)) {
+      _results = parsed.results.slice(-RING_SIZE);
+    }
+  } catch {
+  }
+}
+function persist() {
+  try {
+    if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
+    const payload = { version: 1, results: _results };
+    fs.writeFileSync(PR_SHADOW_FILE, JSON.stringify(payload), "utf8");
+  } catch (err) {
+    logger.warn({ err }, "pr-shadow-store: persist failed");
+  }
+}
+function recordPRShadow(input) {
+  ensureLoaded();
+  const entry = {
+    ...input,
+    id: randomUUID(),
+    timestamp: Date.now()
+  };
+  _results.push(entry);
+  if (_results.length > RING_SIZE) _results.splice(0, _results.length - RING_SIZE);
+  persist();
+  logger.debug(
+    { prId: entry.prId, wouldHaveShown: entry.wouldHaveShown, score: entry.relevanceScore },
+    "pr-shadow: result recorded"
+  );
+  return entry;
+}
+function getPRShadowResults(limit = 100) {
+  ensureLoaded();
+  return _results.slice(-limit).reverse();
+}
+function getPRShadowStats(windowMs = 30 * 24 * 60 * 60 * 1e3) {
+  ensureLoaded();
+  const cutoff = Date.now() - windowMs;
+  const inWindow = _results.filter((r) => r.timestamp >= cutoff);
+  const total = inWindow.length;
+  const shown = inWindow.filter((r) => r.wouldHaveShown).length;
+  const wouldHaveBeenUsefulRate = total === 0 ? null : Math.round(shown / total * 1e3) / 10;
+  const avgRelevanceScore = total === 0 ? null : Math.round(
+    inWindow.reduce((a, r) => a + r.relevanceScore, 0) / total * 1e3
+  ) / 1e3;
+  const triggerCounts = {};
+  for (const r of inWindow) {
+    for (const t of r.triggeredBy) {
+      triggerCounts[t] = (triggerCounts[t] ?? 0) + 1;
+    }
+  }
+  const topTriggers = Object.entries(triggerCounts).sort((a, b) => b[1] - a[1]).map(([trigger, count]) => ({ trigger, count }));
+  return {
+    totalAnalyzed: total,
+    wouldHaveShown: shown,
+    wouldHaveBeenUsefulRate,
+    avgRelevanceScore,
+    topTriggers
+  };
+}
+function _resetPRShadowForTesting() {
+  _results = [];
+  _loaded = true;
+}
+export {
+  _resetPRShadowForTesting,
+  getPRShadowResults,
+  getPRShadowStats,
+  recordPRShadow
+};

package/dist/sensor/process-watcher.js

@@ -0,0 +1,162 @@
+import { spawn } from "child_process";
+import { store } from "./buffer.js";
+import logger from "./logger.js";
+const MAX_LINES_PER_SEC = 30;
+const MAX_LINE_LENGTH = 2e3;
+const RATE_WINDOW_MS = 1e3;
+const ANSI_RE = /\x1b(?:\[[0-9;]*[A-Za-z]|\][^\x07]*(?:\x07|\x1b\\)|\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e])/g;
+function stripAnsi(s) {
+  return s.replace(ANSI_RE, "");
+}
+function frameworkEnv(command, args) {
+  const full = [command, ...args].join(" ").toLowerCase();
+  const env = { NO_COLOR: "1", TERM: "dumb" };
+  if (/python|manage\.py|django|gunicorn|uvicorn|fastapi/.test(full)) {
+    env.PYTHONUNBUFFERED = "1";
+  }
+  if (/\bjava\b|mvn|gradle|spring-boot|bootrun/.test(full)) {
+    const existing = process.env.JAVA_TOOL_OPTIONS ?? "";
+    const flag = "-Dspring.output.ansi.enabled=never";
+    env.JAVA_TOOL_OPTIONS = existing.includes(flag) ? existing : existing ? `${existing} ${flag}` : flag;
+  }
+  if (/\bruby\b|rails|puma|unicorn|rackup/.test(full)) {
+    env.DISABLE_SPRING = "1";
+    env.RAILS_LOG_TO_STDOUT = "1";
+  }
+  return env;
+}
+const TRACEPARENT_RE = /\b00-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}\b/i;
+const TRACE_KV_RE = /(?:trace[_-]?id|traceid)["'\s]*[:=]["'\s]*([0-9a-f]{32})\b/i;
+function extractTraceId(line) {
+  const tp = TRACEPARENT_RE.exec(line);
+  if (tp) return tp[1].toLowerCase();
+  const kv = TRACE_KV_RE.exec(line);
+  if (kv) return kv[1].toLowerCase();
+  return null;
+}
+const _watchers = /* @__PURE__ */ new Map();
+function startProcessWatcher(opts) {
+  const { name, command, args = [], cwd, env } = opts;
+  if (_watchers.has(name)) {
+    logger.warn({ name }, "process watcher already running for this name \u2014 stopping previous");
+    stopProcessWatcher(name);
+  }
+  let proc;
+  try {
+    proc = spawn(command, args, {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+      // frameworkEnv provides safe defaults; caller opts.env always wins
+      env: { ...process.env, ...frameworkEnv(command, args), ...env }
+    });
+  } catch (err) {
+    logger.warn({ name, err }, "process-watcher: failed to spawn process");
+    return;
+  }
+  const watcher = {
+    proc,
+    name,
+    lineCount: 0,
+    windowStart: Date.now(),
+    droppedInWindow: 0
+  };
+  _watchers.set(name, watcher);
+  logger.info({ name, command, args }, "process-watcher: started");
+  function handleLine(raw, isStderr) {
+    const now = Date.now();
+    if (now - watcher.windowStart > RATE_WINDOW_MS) {
+      if (watcher.droppedInWindow > 0) {
+        pushTerminal(watcher, `[mergen] dropped ${watcher.droppedInWindow} lines (rate limit: ${MAX_LINES_PER_SEC}/s)`);
+      }
+      watcher.lineCount = 0;
+      watcher.windowStart = now;
+      watcher.droppedInWindow = 0;
+    }
+    if (watcher.lineCount >= MAX_LINES_PER_SEC) {
+      watcher.droppedInWindow++;
+      return;
+    }
+    watcher.lineCount++;
+    const line = stripAnsi(raw).slice(0, MAX_LINE_LENGTH);
+    const data = isStderr ? `[stderr] ${line}` : line;
+    pushTerminal(watcher, data);
+  }
+  function pushTerminal(w, data) {
+    const traceId = extractTraceId(data) ?? void 0;
+    const event = {
+      type: "terminal",
+      terminalName: w.name,
+      data,
+      timestamp: Date.now(),
+      ...traceId ? { traceId } : {}
+    };
+    try {
+      store.push(event);
+    } catch {
+    }
+  }
+  let stdoutBuf = "";
+  proc.stdout?.on("data", (chunk) => {
+    stdoutBuf += chunk.toString("utf8");
+    const lines = stdoutBuf.split("\n");
+    stdoutBuf = lines.pop() ?? "";
+    for (const l of lines) if (l.trim()) handleLine(l, false);
+  });
+  let stderrBuf = "";
+  proc.stderr?.on("data", (chunk) => {
+    stderrBuf += chunk.toString("utf8");
+    const lines = stderrBuf.split("\n");
+    stderrBuf = lines.pop() ?? "";
+    for (const l of lines) if (l.trim()) handleLine(l, true);
+  });
+  proc.on("error", (err) => {
+    logger.warn({ name, err }, "process-watcher: spawn error");
+    _watchers.delete(name);
+  });
+  proc.on("close", (code, signal) => {
+    _watchers.delete(name);
+    const isCrash = code !== 0 && code !== null;
+    const isOom = code === 137;
+    const reason = isOom ? "oom" : signal ? "signal" : isCrash ? "crash" : "normal";
+    logger.info({ name, code, signal, reason }, "process-watcher: process exited");
+    const exitEvent = {
+      type: "process_exit",
+      process: name,
+      exitCode: code ?? -1,
+      reason,
+      signal: signal ?? void 0,
+      timestamp: Date.now()
+    };
+    try {
+      store.push(exitEvent);
+    } catch {
+    }
+    if (isCrash) {
+      pushTerminal({ ...watcher, name }, `[mergen] process "${name}" exited with code ${code}${signal ? ` (${signal})` : ""}`);
+    }
+  });
+}
+function stopProcessWatcher(name) {
+  const w = _watchers.get(name);
+  if (!w) return;
+  try {
+    w.proc.stdout?.destroy();
+    w.proc.stderr?.destroy();
+    w.proc.kill();
+  } catch {
+  }
+  _watchers.delete(name);
+  logger.info({ name }, "process-watcher: stopped");
+}
+function stopAllProcessWatchers() {
+  for (const name of _watchers.keys()) stopProcessWatcher(name);
+}
+function listProcessWatchers() {
+  return [..._watchers.keys()];
+}
+export {
+  listProcessWatchers,
+  startProcessWatcher,
+  stopAllProcessWatchers,
+  stopProcessWatcher
+};

package/dist/sensor/rbac.js

@@ -0,0 +1,92 @@
+import fs from "fs";
+import { RBAC_FILE, DATA_DIR } from "./paths.js";
+import logger from "./logger.js";
+const ROLE_RANK = { viewer: 0, responder: 1, admin: 2 };
+let _loadFailed = false;
+function load() {
+  if (!fs.existsSync(RBAC_FILE)) {
+    _loadFailed = false;
+    return { members: [] };
+  }
+  try {
+    const raw = fs.readFileSync(RBAC_FILE, "utf8");
+    const parsed = JSON.parse(raw);
+    if (typeof parsed !== "object" || parsed === null || !Array.isArray(parsed.members)) {
+      _loadFailed = true;
+      logger.error({ path: RBAC_FILE }, 'rbac: RBAC file is malformed (missing/invalid "members" array) \u2014 failing closed (all actors restricted to viewer)');
+      return { members: [] };
+    }
+    _loadFailed = false;
+    return parsed;
+  } catch (err) {
+    _loadFailed = true;
+    logger.error({ err }, "rbac: RBAC file is unreadable \u2014 failing closed (all actors restricted to viewer)");
+    return { members: [] };
+  }
+}
+function save(store) {
+  try {
+    fs.mkdirSync(DATA_DIR, { recursive: true });
+    fs.writeFileSync(RBAC_FILE, JSON.stringify(store, null, 2), "utf8");
+    _loadFailed = false;
+  } catch (err) {
+    logger.warn({ err }, "rbac: failed to write rbac.json");
+  }
+}
+function listMembers() {
+  return load().members;
+}
+function upsertMember(id, role, source = "manual") {
+  const store = load();
+  const existing = store.members.find((m) => m.id === id);
+  if (existing) {
+    if (source === "oidc-sync" && (existing.source ?? "manual") === "manual") {
+      logger.info(
+        { id, pinnedRole: existing.role, syncedRole: role },
+        "rbac: OIDC group-role sync skipped \u2014 member role is manually pinned"
+      );
+      return { id: existing.id, role: existing.role, source: existing.source };
+    }
+    existing.role = role;
+    existing.source = source;
+  } else {
+    store.members.push({ id, role, source });
+  }
+  save(store);
+  logger.info({ id, role, source }, "rbac: member upserted");
+  return { id, role, source };
+}
+function removeMember(id) {
+  const store = load();
+  const before = store.members.length;
+  store.members = store.members.filter((m) => m.id !== id);
+  if (store.members.length < before) {
+    save(store);
+    logger.info({ id }, "rbac: member removed");
+    return true;
+  }
+  return false;
+}
+function resolveRole(actor) {
+  if (actor === "autopilot") return "admin";
+  const store = load();
+  if (_loadFailed) return "viewer";
+  if (store.members.length === 0) return "admin";
+  const member = store.members.find((m) => m.id === actor);
+  return member?.role ?? "viewer";
+}
+function hasPermission(actor, required) {
+  const actual = resolveRole(actor);
+  return ROLE_RANK[actual] >= ROLE_RANK[required];
+}
+function _resetForTesting() {
+  _loadFailed = false;
+}
+export {
+  _resetForTesting,
+  hasPermission,
+  listMembers,
+  removeMember,
+  resolveRole,
+  upsertMember
+};

package/dist/sensor/redact.js

@@ -0,0 +1,114 @@
+const DEFAULT_KEYS = [
+  "password",
+  "passwd",
+  "pwd",
+  "authorization",
+  "auth",
+  "cookie",
+  "set-cookie",
+  "token",
+  "access_token",
+  "refresh_token",
+  "id_token",
+  "jwt",
+  "apikey",
+  "api_key",
+  "api-key",
+  "secret",
+  "client_secret",
+  "session",
+  "sessionid",
+  "session_id",
+  "email",
+  "ssn",
+  "creditcard",
+  "credit_card",
+  "cc_number"
+];
+function loadKeys() {
+  const env = process.env.MERGEN_REDACT_KEYS;
+  const extra = env ? env.split(",").map((s) => s.trim().toLowerCase()).filter(Boolean) : [];
+  return /* @__PURE__ */ new Set([...DEFAULT_KEYS, ...extra]);
+}
+let _keys = loadKeys();
+function reloadRedactKeys() {
+  _keys = loadKeys();
+}
+function reloadPiiConfig() {
+  loadPiiConfig();
+}
+const REDACTED = "[REDACTED]";
+const MAX_DEPTH = 8;
+const RE_JWT = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g;
+const RE_BEARER = /\bBearer\s+[A-Za-z0-9._\-+/=]{8,}\b/gi;
+const RE_EMAIL = /\b[\w.+-]+@[\w-]+\.[\w.-]+\b/g;
+const RE_CARD = /\b(?:\d{4}[ -]?\d{4}[ -]?\d{4}[ -]?\d{1,7}|\d{4}[ -]?\d{6}[ -]?\d{5})\b/g;
+const RE_PHONE = /\b(\+?1[\s.-]?)?\(?[2-9]\d{2}\)?[\s.-]?\d{3}[\s.-]\d{4}\b/g;
+const RE_AWS_KEY = /\b(AKIA|AROA|ASIA|AIDA|ANPA|ANVA|APKA)[0-9A-Z]{16}\b/g;
+const RE_PEM = /-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----/g;
+const RE_STRIPE = /\b(?:sk|pk|rk)_(?:live|test)_[A-Za-z0-9]{24,}\b/g;
+const RE_CONN_STR = /\b([a-zA-Z0-9+.-]+:\/\/)([^:\s]+):([^@\s]+)@([^\s/]+)/gi;
+import fs from "fs";
+import path from "path";
+import os from "os";
+const PII_CONFIG_FILE = path.join(os.homedir(), ".mergen", "pii-config.json");
+let _customPatterns = [];
+function loadPiiConfig() {
+  if (!fs.existsSync(PII_CONFIG_FILE)) return;
+  try {
+    const raw = JSON.parse(fs.readFileSync(PII_CONFIG_FILE, "utf8"));
+    _customPatterns = (raw.patterns ?? []).flatMap((p) => {
+      try {
+        const m = p.match(/^\/(.*)\/([gimsuy]*)$/);
+        return m ? [new RegExp(m[1], m[2])] : [];
+      } catch {
+        return [];
+      }
+    });
+  } catch {
+  }
+}
+loadPiiConfig();
+function redactString(s) {
+  if (s.length > 1e5) return REDACTED;
+  let out = s.replace(RE_PEM, REDACTED).replace(RE_JWT, REDACTED).replace(RE_BEARER, `Bearer ${REDACTED}`).replace(RE_AWS_KEY, REDACTED).replace(RE_STRIPE, REDACTED).replace(RE_CONN_STR, "$1$2:[REDACTED]@$4").replace(RE_EMAIL, REDACTED).replace(RE_CARD, REDACTED).replace(RE_PHONE, REDACTED);
+  for (const re of _customPatterns) {
+    re.lastIndex = 0;
+    out = out.replace(re, REDACTED);
+  }
+  return out;
+}
+function isSensitiveKey(key) {
+  return _keys.has(key.toLowerCase());
+}
+function redactInternal(value, depth) {
+  if (depth > MAX_DEPTH) return value;
+  if (value === null || value === void 0) return value;
+  const t = typeof value;
+  if (t === "string") return redactString(value);
+  if (t === "number" || t === "boolean" || t === "bigint") return value;
+  if (Array.isArray(value)) {
+    return value.map((v) => redactInternal(v, depth + 1));
+  }
+  if (t === "object") {
+    const obj = value;
+    const out = {};
+    for (const k of Object.keys(obj)) {
+      out[k] = isSensitiveKey(k) ? REDACTED : redactInternal(obj[k], depth + 1);
+    }
+    return out;
+  }
+  return value;
+}
+function redact(value) {
+  try {
+    return redactInternal(value, 0);
+  } catch {
+    return value;
+  }
+}
+export {
+  redact,
+  reloadPiiConfig,
+  reloadRedactKeys
+};