mergen-server 1.4.0

package/dist/sensor/action-ledger.js

@@ -0,0 +1,327 @@
+import initSqlJs from "sql.js";
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { createRequire } from "module";
+import { DATA_DIR } from "./paths.js";
+import logger from "./logger.js";
+const LEDGER_DB = path.join(DATA_DIR, "action-ledger.db");
+const RETENTION_MS = (() => {
+  const d = parseFloat(process.env.MERGEN_LEDGER_RETENTION_DAYS ?? "90");
+  return (Number.isFinite(d) && d > 0 ? d : 90) * 24 * 60 * 60 * 1e3;
+})();
+const MAX_ROWS = 1e6;
+const FLUSH_DEBOUNCE_MS = 2e3;
+class ActionLedger {
+  db = null;
+  wasmPath;
+  writeCount = 0;
+  flushTimer = null;
+  // Rows appended before init() finishes are held here and drained on init.
+  preInitQueue = [];
+  constructor() {
+    this.wasmPath = ActionLedger.resolveWasmPath();
+  }
+  static resolveWasmPath() {
+    if (process.env.MERGEN_WASM_PATH) return process.env.MERGEN_WASM_PATH;
+    const moduleDir = path.dirname(fileURLToPath(import.meta.url));
+    const fromModule = path.resolve(moduleDir, "../../node_modules/sql.js/dist/sql-wasm.wasm");
+    if (fs.existsSync(fromModule)) return fromModule;
+    try {
+      const req = createRequire(import.meta.url);
+      const resolved = path.join(path.dirname(req.resolve("sql.js")), "sql-wasm.wasm");
+      if (fs.existsSync(resolved)) return resolved;
+    } catch {
+    }
+    return fromModule;
+  }
+  async init() {
+    try {
+      const wasmBinary = fs.readFileSync(this.wasmPath);
+      const SQL = await initSqlJs({ wasmBinary });
+      fs.mkdirSync(DATA_DIR, { recursive: true });
+      let fileBuffer;
+      if (fs.existsSync(LEDGER_DB)) {
+        try {
+          fileBuffer = fs.readFileSync(LEDGER_DB);
+        } catch {
+          logger.warn("action-ledger.db unreadable, starting fresh");
+        }
+      }
+      this.db = fileBuffer ? new SQL.Database(fileBuffer) : new SQL.Database();
+      this.db.run(`
+        CREATE TABLE IF NOT EXISTS action_ledger (
+          id                INTEGER PRIMARY KEY AUTOINCREMENT,
+          ts                INTEGER NOT NULL,
+          agent_id          TEXT,
+          user_id           TEXT,
+          tool              TEXT    NOT NULL,
+          command           TEXT,
+          resource          TEXT,
+          verdict           TEXT    NOT NULL,
+          policy_matched    TEXT    NOT NULL DEFAULT '[]',
+          reason            TEXT,
+          credential_issued INTEGER NOT NULL DEFAULT 0,
+          credential_scope  TEXT,
+          environment       TEXT,
+          trace_id          TEXT
+        );
+        CREATE INDEX IF NOT EXISTS idx_ledger_agent_ts ON action_ledger (agent_id, ts);
+        CREATE INDEX IF NOT EXISTS idx_ledger_ts       ON action_ledger (ts);
+        CREATE INDEX IF NOT EXISTS idx_ledger_cred     ON action_ledger (credential_scope, environment, ts);
+      `);
+      this.flush();
+      const pending = this.preInitQueue;
+      this.preInitQueue = [];
+      for (const rec of pending) this.appendActionRecord(rec);
+      logger.info({ path: LEDGER_DB, retentionDays: RETENTION_MS / 864e5 }, "action-ledger: initialised");
+      this.scheduleRetention();
+    } catch (err) {
+      logger.warn(
+        { err, wasmPath: this.wasmPath },
+        "action-ledger: init failed \u2014 durable action history unavailable. Set MERGEN_WASM_PATH to override the wasm location."
+      );
+      this.db = null;
+    }
+  }
+  /** Append a decision. Fire-and-forget from the gate path: never throws to the
+   *  caller, never blocks on disk (the fsync is debounced + async). */
+  appendActionRecord(rec) {
+    try {
+      if (!this.db) {
+        if (this.preInitQueue.length < 1e4) this.preInitQueue.push(rec);
+        return;
+      }
+      this.db.run(
+        `INSERT INTO action_ledger
+           (ts, agent_id, user_id, tool, command, resource, verdict,
+            policy_matched, reason, credential_issued, credential_scope, environment, trace_id)
+         VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+        [
+          rec.ts ?? Date.now(),
+          rec.agentId ?? null,
+          rec.userId ?? null,
+          rec.tool,
+          rec.command ?? null,
+          rec.resource ?? null,
+          rec.verdict,
+          JSON.stringify(rec.policyMatched ?? []),
+          rec.reason ?? null,
+          rec.credentialIssued ? 1 : 0,
+          rec.credentialScope ?? null,
+          rec.environment ?? null,
+          rec.traceId ?? null
+        ]
+      );
+      this.writeCount++;
+      this.scheduleFlush();
+      this.forwardSiem(rec);
+    } catch (err) {
+      logger.warn({ err }, "action-ledger: append failed");
+    }
+  }
+  forwardSiem(rec) {
+    let id = 0;
+    try {
+      id = this.db.exec("SELECT last_insert_rowid()")[0]?.values[0]?.[0] ?? 0;
+    } catch {
+    }
+    void import("../intelligence/siem-forward.js").then(({ forwardLedgerToSiem }) => {
+      forwardLedgerToSiem({
+        id,
+        ts: rec.ts ?? Date.now(),
+        agentId: rec.agentId ?? null,
+        userId: rec.userId ?? null,
+        tool: rec.tool,
+        command: rec.command ?? null,
+        resource: rec.resource ?? null,
+        verdict: rec.verdict,
+        policyMatched: rec.policyMatched ?? [],
+        reason: rec.reason ?? null,
+        credentialIssued: rec.credentialIssued ?? false,
+        credentialScope: rec.credentialScope ?? null,
+        environment: rec.environment ?? null,
+        traceId: rec.traceId ?? null
+      });
+    }).catch(() => {
+    });
+  }
+  queryLedger(opts = {}) {
+    if (!this.db) return [];
+    try {
+      const { agentId, from, to, verdict, tool, limit = 100 } = opts;
+      const clauses = ["1=1"];
+      const params = [];
+      if (agentId) {
+        clauses.push("agent_id = ?");
+        params.push(agentId);
+      }
+      if (from !== void 0) {
+        clauses.push("ts >= ?");
+        params.push(from);
+      }
+      if (to !== void 0) {
+        clauses.push("ts <= ?");
+        params.push(to);
+      }
+      if (verdict) {
+        clauses.push("verdict = ?");
+        params.push(verdict);
+      }
+      if (tool) {
+        clauses.push("tool = ?");
+        params.push(tool);
+      }
+      const sql = `SELECT * FROM action_ledger WHERE ${clauses.join(" AND ")} ORDER BY ts DESC LIMIT ?`;
+      params.push(Math.min(Math.max(1, limit), 5e3));
+      return this.readRows(sql, params);
+    } catch (err) {
+      logger.warn({ err }, "action-ledger: query failed");
+      return [];
+    }
+  }
+  /** Credential-issuance rows only — feeds cross-environment misuse detection. */
+  queryCredentialUsage(opts = {}) {
+    if (!this.db) return [];
+    try {
+      const { agentId, scope, from, to, limit = 1e3 } = opts;
+      const clauses = ["credential_issued = 1"];
+      const params = [];
+      if (agentId) {
+        clauses.push("agent_id = ?");
+        params.push(agentId);
+      }
+      if (scope) {
+        clauses.push("credential_scope = ?");
+        params.push(scope);
+      }
+      if (from !== void 0) {
+        clauses.push("ts >= ?");
+        params.push(from);
+      }
+      if (to !== void 0) {
+        clauses.push("ts <= ?");
+        params.push(to);
+      }
+      const sql = `SELECT * FROM action_ledger WHERE ${clauses.join(" AND ")} ORDER BY ts DESC LIMIT ?`;
+      params.push(Math.min(Math.max(1, limit), 1e4));
+      return this.readRows(sql, params);
+    } catch (err) {
+      logger.warn({ err }, "action-ledger: credential query failed");
+      return [];
+    }
+  }
+  readRows(sql, params) {
+    const stmt = this.db.prepare(sql);
+    stmt.bind(params);
+    const rows = [];
+    while (stmt.step()) {
+      const r = stmt.getAsObject();
+      let policyMatched = [];
+      try {
+        policyMatched = JSON.parse(r["policy_matched"] ?? "[]");
+      } catch {
+      }
+      rows.push({
+        id: r["id"],
+        ts: r["ts"],
+        agentId: r["agent_id"] ?? null,
+        userId: r["user_id"] ?? null,
+        tool: r["tool"],
+        command: r["command"] ?? null,
+        resource: r["resource"] ?? null,
+        verdict: r["verdict"],
+        policyMatched,
+        reason: r["reason"] ?? null,
+        credentialIssued: r["credential_issued"] === 1,
+        credentialScope: r["credential_scope"] ?? null,
+        environment: r["environment"] ?? null,
+        traceId: r["trace_id"] ?? null
+      });
+    }
+    stmt.free();
+    return rows;
+  }
+  isHealthy() {
+    return this.db !== null;
+  }
+  /** Delete rows past the retention window, then enforce the hard row cap.
+   *  Called on init, hourly, and (in tests) directly. */
+  prune() {
+    if (!this.db) return;
+    try {
+      this.db.run("DELETE FROM action_ledger WHERE ts < ?", [Date.now() - RETENTION_MS]);
+      const total = this.db.exec("SELECT COUNT(*) FROM action_ledger")[0]?.values[0]?.[0] ?? 0;
+      if (total > MAX_ROWS) {
+        const excess = total - MAX_ROWS;
+        this.db.run("DELETE FROM action_ledger WHERE id IN (SELECT id FROM action_ledger ORDER BY ts ASC LIMIT ?)", [excess]);
+        logger.warn({ excess }, "action-ledger: row cap exceeded \u2014 pruned oldest rows");
+      }
+      this.flush();
+    } catch (err) {
+      logger.warn({ err }, "action-ledger: retention prune failed");
+    }
+  }
+  scheduleRetention() {
+    this.prune();
+    setInterval(() => this.prune(), 60 * 6e4).unref();
+  }
+  // ── Disk persistence (async tmp+rename, coalesced) ──────────────────────────
+  scheduleFlush() {
+    if (this.flushTimer) return;
+    this.flushTimer = setTimeout(() => {
+      this.flushTimer = null;
+      this.flush();
+    }, FLUSH_DEBOUNCE_MS);
+    if (this.flushTimer && typeof this.flushTimer === "object" && "unref" in this.flushTimer) {
+      this.flushTimer.unref();
+    }
+  }
+  isWriting = false;
+  pendingWrite = false;
+  nextBuffer = null;
+  /** Force a synchronous-enough flush; also used at graceful shutdown. */
+  flush() {
+    if (!this.db) return;
+    try {
+      const data = this.db.export();
+      this.writeBufferAsync(Buffer.from(data.buffer, data.byteOffset, data.byteLength));
+    } catch (err) {
+      logger.warn({ err }, "action-ledger: flush failed");
+    }
+  }
+  writeBufferAsync(buf) {
+    if (this.isWriting) {
+      this.pendingWrite = true;
+      this.nextBuffer = buf;
+      return;
+    }
+    this.isWriting = true;
+    const tmp = `${LEDGER_DB}.tmp.${process.pid}`;
+    fs.writeFile(tmp, buf, (err) => {
+      if (err) {
+        logger.warn({ err }, "action-ledger: async write failed");
+        this.isWriting = false;
+        this.processPendingWrite();
+        return;
+      }
+      fs.rename(tmp, LEDGER_DB, (renameErr) => {
+        if (renameErr) logger.warn({ err: renameErr }, "action-ledger: async rename failed");
+        this.isWriting = false;
+        this.processPendingWrite();
+      });
+    });
+  }
+  processPendingWrite() {
+    if (this.pendingWrite && this.nextBuffer) {
+      this.pendingWrite = false;
+      const buf = this.nextBuffer;
+      this.nextBuffer = null;
+      this.writeBufferAsync(buf);
+    }
+  }
+}
+const actionLedger = new ActionLedger();
+export {
+  actionLedger
+};

package/dist/sensor/adr-store.js

@@ -0,0 +1,140 @@
+import fs from "fs";
+import path from "path";
+import { DATA_DIR, zeroRetentionMode } from "./paths.js";
+import logger from "./logger.js";
+const ADR_FILE = path.join(DATA_DIR, "adrs.json");
+const SEED_ADRS = [
+  {
+    id: "ADR-001",
+    title: "Ring buffer as primary event store",
+    status: "accepted",
+    date: "2024-01-15",
+    decision: "Use a fixed-capacity (2 000-event) in-memory ring buffer as the primary store for inbound telemetry events, with O(1) eviction of the oldest event when the cap is reached.",
+    alternatives: [
+      "Unlimited in-memory list \u2014 rejected: unbounded memory growth would cause OOM crashes",
+      "Write-through to SQLite on every event \u2014 rejected: synchronous disk I/O bottlenecks throughput during incident spikes",
+      "Redis as primary store \u2014 rejected: adds an external dependency that breaks the zero-infrastructure install story"
+    ],
+    rationale: "Incident triage tools need the most recent events, not a complete historical ledger. A ring buffer keeps the hot path allocation-free, provides constant-time reads and writes, and caps memory at a predictable ceiling.",
+    consequences: "Reading more than 2 000 events requires querying the SQLite history store. The cap is tunable via MERGEN_BUFFER_SIZE env var."
+  },
+  {
+    id: "ADR-002",
+    title: "MCP protocol over custom REST for AI IDE integration",
+    status: "accepted",
+    date: "2024-02-01",
+    decision: "Expose Mergen's analysis and triage capabilities as Model Context Protocol (MCP) tools over stdio transport, rather than a proprietary REST API.",
+    alternatives: [
+      "Custom REST API with IDE plugins \u2014 rejected: every IDE needs a bespoke plugin; integration surface multiplies with each new IDE",
+      "Language Server Protocol (LSP) \u2014 rejected: LSP is designed for code intelligence, not arbitrary tool invocation",
+      "OpenAI function-calling format \u2014 rejected: not IDE-agnostic; couples Mergen to a single model vendor"
+    ],
+    rationale: "MCP is the emerging standard for AI tool exposure. A single MCP server declaration in a project config file immediately surfaces all Mergen tools in every compatible IDE.",
+    consequences: "The HTTP server (:3000) and MCP server (stdio) are separate processes sharing state via the in-memory buffer. MCP version upgrades may require updating tool registration patterns."
+  },
+  {
+    id: "ADR-003",
+    title: "SQLite for persistent event history",
+    status: "accepted",
+    date: "2024-02-15",
+    decision: "Use SQLite (via sql.js) for the 1-hour persistent event history layer, rather than a networked database or plain JSON files.",
+    alternatives: [
+      "Plain JSON files \u2014 rejected: not atomic; crash mid-write corrupts the file; range queries require full deserialisation",
+      "PostgreSQL / MySQL \u2014 rejected: requires a running database server, breaking the zero-infrastructure install story",
+      "Redis \u2014 rejected: persistence config complexity; not enabled by default",
+      "LevelDB / RocksDB \u2014 rejected: native bindings fail on ARM Macs and Alpine Linux"
+    ],
+    rationale: "sql.js compiles SQLite to WebAssembly \u2014 zero native bindings, consistent binary on every platform. SQLite provides ACID guarantees and efficient timestamp range scans.",
+    consequences: "The sql.js WASM binary adds ~1.2 MB to the installed package. Write throughput is single-threaded. Schema migrations must be handled manually in sqlite-store.ts."
+  },
+  {
+    id: "ADR-004",
+    title: "Ingest server binds to 127.0.0.1 by default",
+    status: "accepted",
+    date: "2024-03-01",
+    decision: "The Express HTTP server listens on 127.0.0.1:3000 (loopback) by default, not 0.0.0.0.",
+    alternatives: [
+      "Bind to 0.0.0.0 by default \u2014 rejected: any process on the local network could read the event buffer or trigger autonomous fix execution",
+      "Bind to 0.0.0.0 with mandatory API key \u2014 rejected: key management burden defeats the zero-config local install goal",
+      "Unix domain socket \u2014 rejected: not supported on Windows"
+    ],
+    rationale: "Developer tools should be safe by default. Loopback binding limits the attack surface to processes already on the same machine. Teams needing network access can set MERGEN_HOST=0.0.0.0.",
+    consequences: "Browser extensions and SDKs must use 127.0.0.1, not localhost. Docker deployments require --network=host or explicit port mapping. Cloud mode overrides this to 0.0.0.0 with TLS."
+  },
+  {
+    id: "ADR-005",
+    title: "Three-tier tool access model (free / pro / all)",
+    status: "accepted",
+    date: "2024-04-01",
+    decision: "Classify every MCP tool into free, pro, or all tiers enforced via withTierGate(). The canonical tier for each tool is declared in tool-manifest.ts.",
+    alternatives: [
+      "Binary free / paid split \u2014 rejected: too coarse; some read-only tools are safe to offer free while execution tools carry risk",
+      "Per-tool pricing \u2014 rejected: metering complexity and unpredictable bills",
+      "No gating at all \u2014 rejected: unrestricted execution tools represent liability without a paid support contract"
+    ],
+    rationale: "Three tiers maps onto what engineers actually need: free for read-only analysis, pro for execution and third-party API integration, all for tools with no restriction.",
+    consequences: "New tools must be added to tool-manifest.ts with a tier before registration. The manifest consistency test will fail otherwise. withTierGate() returns an upgrade prompt rather than a 403."
+  }
+];
+class AdrStore {
+  records = /* @__PURE__ */ new Map();
+  constructor() {
+    for (const adr of SEED_ADRS) {
+      this.records.set(adr.id, adr);
+    }
+    this._loadFromDisk();
+  }
+  _loadFromDisk() {
+    try {
+      if (!fs.existsSync(ADR_FILE)) return;
+      const raw = fs.readFileSync(ADR_FILE, "utf8");
+      const extra = JSON.parse(raw);
+      for (const adr of extra) {
+        if (adr.id && !this.records.has(adr.id)) {
+          this.records.set(adr.id, adr);
+        }
+      }
+    } catch {
+    }
+  }
+  _saveToDisk() {
+    if (zeroRetentionMode()) return;
+    try {
+      if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
+      const userAdrs = [...this.records.values()].filter(
+        (r) => !SEED_ADRS.some((s) => s.id === r.id)
+      );
+      fs.writeFileSync(ADR_FILE, JSON.stringify(userAdrs, null, 2), "utf8");
+    } catch (err) {
+      logger.warn({ err }, "adr-store: failed to persist to disk");
+    }
+  }
+  list(query) {
+    const all = [...this.records.values()];
+    if (!query) return all;
+    const q = query.toLowerCase();
+    return all.filter(
+      (r) => r.id.toLowerCase().includes(q) || r.title.toLowerCase().includes(q) || r.decision.toLowerCase().includes(q) || r.rationale.toLowerCase().includes(q) || r.alternatives.some((a) => a.toLowerCase().includes(q))
+    );
+  }
+  get(id) {
+    return this.records.get(id.toUpperCase());
+  }
+  add(record) {
+    const nextNum = this.records.size + 1;
+    const id = `ADR-${String(nextNum).padStart(3, "0")}`;
+    const adr = { id, ...record };
+    this.records.set(id, adr);
+    this._saveToDisk();
+    logger.info({ id }, "adr-store: new ADR recorded");
+    return adr;
+  }
+  upsert(record) {
+    this.records.set(record.id.toUpperCase(), record);
+    this._saveToDisk();
+  }
+}
+const adrStore = new AdrStore();
+export {
+  adrStore
+};