mergen-server 1.4.0

package/dist/sensor/agent-blunder-store.js

@@ -0,0 +1,370 @@
+import fs from "fs";
+import path from "path";
+import { createHash, createHmac, randomUUID, randomBytes, timingSafeEqual } from "crypto";
+import { DATA_DIR, zeroRetentionMode } from "./paths.js";
+import logger from "./logger.js";
+import { lockAndExecute } from "./file-lock.js";
+const BLUNDER_FILE = path.join(DATA_DIR, "agent-blunders.json");
+const BLUNDER_HMAC_FILE = BLUNDER_FILE + ".hmac";
+const CHECKPOINT_FILE = path.join(DATA_DIR, "agent-blunders.checkpoints.jsonl");
+const CHECKPOINT_INTERVAL = 50;
+let _blunderHmacSecret = "";
+let _blunderHmacSecretConfigured = false;
+function setBlunderHmacSecret(secret) {
+  _blunderHmacSecret = secret;
+  _blunderHmacSecretConfigured = true;
+}
+function _resetBlunderHmacSecretForTesting() {
+  _blunderHmacSecret = "";
+  _blunderHmacSecretConfigured = false;
+}
+function _blunderHmacKey() {
+  return process.env.MERGEN_AUDIT_SECRET || _blunderHmacSecret;
+}
+function hmacKeyConfigured() {
+  return !!(process.env.MERGEN_AUDIT_SECRET || _blunderHmacSecretConfigured);
+}
+function _writeHmac(contents) {
+  if (!hmacKeyConfigured()) return;
+  try {
+    const hmac = createHmac("sha256", _blunderHmacKey()).update(contents).digest("hex");
+    fs.writeFileSync(BLUNDER_HMAC_FILE, hmac, "utf8");
+  } catch {
+  }
+}
+function _verifyHmac(contents) {
+  if (!hmacKeyConfigured()) return true;
+  try {
+    const stored = fs.readFileSync(BLUNDER_HMAC_FILE, "utf8").trim();
+    const expected = createHmac("sha256", _blunderHmacKey()).update(contents).digest("hex");
+    const storedBuf = Buffer.from(stored, "hex");
+    const expectedBuf = Buffer.from(expected, "hex");
+    if (storedBuf.length !== expectedBuf.length) return false;
+    return timingSafeEqual(storedBuf, expectedBuf);
+  } catch {
+    return true;
+  }
+}
+function emitCheckpoint(latest) {
+  const checkpoint = {
+    sequenceNumber: latest.sequenceNumber,
+    hash: latest.hash,
+    timestamp: Date.now()
+  };
+  logger.info({ checkpoint }, "agent-blunder-store: audit checkpoint");
+  if (zeroRetentionMode() || _testingMode || process.env.VITEST) return;
+  try {
+    fs.mkdirSync(DATA_DIR, { recursive: true, mode: 448 });
+    fs.appendFileSync(CHECKPOINT_FILE, JSON.stringify(checkpoint) + "\n", { encoding: "utf8", flag: "a" });
+  } catch (err) {
+    logger.warn({ err }, "agent-blunder-store: checkpoint sidecar write failed (non-fatal)");
+  }
+}
+function tamperEvidenceLevel(chainHasVerifiedEntries) {
+  if (!chainHasVerifiedEntries) return "none";
+  return hmacKeyConfigured() ? "hmac-sealed" : "hash-chain";
+}
+const MAX_BLUNDERS = (() => {
+  const v = parseInt(process.env.MERGEN_MAX_BLUNDERS ?? "5000", 10);
+  return Number.isFinite(v) && v > 0 ? Math.min(v, 1e5) : 5e3;
+})();
+const GENESIS_HASH = "0".repeat(64);
+let _blunders = [];
+let _loaded = false;
+let _nextSequenceNumber = 0;
+let _testingMode = false;
+function _hashableContent(event, previousHash) {
+  const content = {
+    id: event.id,
+    recordedAt: event.recordedAt,
+    blunderType: event.blunderType,
+    command: event.command,
+    blockReason: event.blockReason,
+    service: event.service,
+    tag: event.tag,
+    actor: event.actor,
+    pid: event.pid,
+    confidenceScore: event.confidenceScore
+  };
+  if (event.triggeredRules !== void 0) content.triggeredRules = event.triggeredRules;
+  if (event.agentId !== void 0) content.agentId = event.agentId;
+  if (event.parentAgentId !== void 0) content.parentAgentId = event.parentAgentId;
+  if (event.humanOwner !== void 0) content.humanOwner = event.humanOwner;
+  if (event.sessionId !== void 0) content.sessionId = event.sessionId;
+  if (event.traceId !== void 0) content.traceId = event.traceId;
+  if (event.sequenceNumber !== void 0) content.sequenceNumber = event.sequenceNumber;
+  return previousHash + JSON.stringify(content);
+}
+function _computeHash(event, previousHash) {
+  return createHash("sha256").update(_hashableContent(event, previousHash)).digest("hex");
+}
+let _integrityViolated = false;
+function isBlunderIntegrityViolated() {
+  return _integrityViolated;
+}
+function load(force = false) {
+  if (_testingMode) return;
+  if (_loaded && !force) return;
+  _loaded = true;
+  if (!fs.existsSync(BLUNDER_FILE)) {
+    _blunders = [];
+    return;
+  }
+  try {
+    const contents = fs.readFileSync(BLUNDER_FILE, "utf8");
+    if (!_verifyHmac(contents)) {
+      _integrityViolated = true;
+      logger.error({ path: BLUNDER_FILE }, "agent-blunder-store: HMAC mismatch \u2014 file may have been tampered with");
+    }
+    const raw = JSON.parse(contents);
+    if ((raw?.version === 3 || raw?.version === 2 || raw?.version === 1) && Array.isArray(raw.blunders)) {
+      _blunders = raw.blunders;
+      const maxSeen = _blunders.reduce((max, b) => typeof b.sequenceNumber === "number" ? Math.max(max, b.sequenceNumber) : max, -1);
+      _nextSequenceNumber = typeof raw.nextSequenceNumber === "number" ? raw.nextSequenceNumber : maxSeen + 1;
+    } else {
+      _blunders = [];
+      _nextSequenceNumber = 0;
+    }
+  } catch {
+    _blunders = [];
+    _nextSequenceNumber = 0;
+  }
+}
+function persist() {
+  if (zeroRetentionMode()) return;
+  try {
+    fs.mkdirSync(DATA_DIR, { recursive: true, mode: 448 });
+    const tmp = `${BLUNDER_FILE}.tmp.${process.pid}.${randomBytes(4).toString("hex")}`;
+    const contents = JSON.stringify({ version: 3, blunders: _blunders, nextSequenceNumber: _nextSequenceNumber });
+    fs.writeFileSync(tmp, contents, "utf8");
+    fs.renameSync(tmp, BLUNDER_FILE);
+    _writeHmac(contents);
+  } catch (err) {
+    logger.warn({ err }, "agent-blunder-store: persist failed");
+  }
+}
+function _resetForTesting() {
+  _blunders = [];
+  _loaded = true;
+  _testingMode = true;
+  _nextSequenceNumber = 0;
+}
+function _injectRawForTesting(entry) {
+  _blunders.push({
+    command: null,
+    service: null,
+    tag: null,
+    actor: null,
+    pid: null,
+    confidenceScore: null,
+    previousHash: "",
+    hash: "",
+    ...entry
+  });
+}
+function recordBlunder(event) {
+  return lockAndExecute(`${BLUNDER_FILE}.lock`, () => {
+    load(true);
+    const id = event.id ?? randomUUID();
+    if (_blunders.some((b) => b.id === id)) {
+      return;
+    }
+    const base = {
+      ...event,
+      id,
+      recordedAt: event.recordedAt ?? Date.now(),
+      // Normalize to null so every new entry carries the field explicitly —
+      // presence distinguishes v3 entries from pre-migration v2 ones.
+      triggeredRules: event.triggeredRules ?? null,
+      // Agent identity — normalize undefined → null so presence-aware hashing
+      // works correctly: new entries always carry these fields explicitly.
+      agentId: event.agentId ?? null,
+      parentAgentId: event.parentAgentId ?? null,
+      humanOwner: event.humanOwner ?? null,
+      sessionId: event.sessionId ?? null,
+      traceId: event.traceId ?? null,
+      sequenceNumber: _nextSequenceNumber
+    };
+    const previousHash = _blunders.length > 0 ? _blunders[_blunders.length - 1].hash || GENESIS_HASH : GENESIS_HASH;
+    const hash = _computeHash(base, previousHash);
+    const entry = { ...base, previousHash, hash };
+    _blunders.push(entry);
+    _nextSequenceNumber++;
+    if (_blunders.length > MAX_BLUNDERS) _blunders = _blunders.slice(-MAX_BLUNDERS);
+    persist();
+    if (entry.sequenceNumber % CHECKPOINT_INTERVAL === 0) emitCheckpoint(entry);
+    void import("../intelligence/siem-forward.js").then(({ forwardToSiem }) => forwardToSiem(entry)).catch(() => {
+    });
+    logger.info(
+      { blunderType: event.blunderType, cmd: event.command?.slice(0, 80), service: event.service, agentId: event.agentId },
+      "agent-blunder: intercepted"
+    );
+  });
+}
+function getBlunders(filter) {
+  load(true);
+  if (filter?.agentId) {
+    return _blunders.filter((b) => b.agentId === filter.agentId);
+  }
+  return [..._blunders];
+}
+const NON_BLOCKING_BLUNDER_TYPES = /* @__PURE__ */ new Set(["credential_issued", "policy_integrity_failure"]);
+function isNonBlockingBlunderType(blunderType) {
+  return NON_BLOCKING_BLUNDER_TYPES.has(blunderType);
+}
+function getBlunderStats() {
+  load(true);
+  const now = Date.now();
+  const ms7 = 7 * 24 * 60 * 60 * 1e3;
+  const ms30 = 30 * 24 * 60 * 60 * 1e3;
+  const byType = {};
+  let total = 0, last7Days = 0, last30Days = 0;
+  for (const b of _blunders) {
+    byType[b.blunderType] = (byType[b.blunderType] ?? 0) + 1;
+    if (NON_BLOCKING_BLUNDER_TYPES.has(b.blunderType)) continue;
+    total++;
+    if (b.recordedAt >= now - ms7) last7Days++;
+    if (b.recordedAt >= now - ms30) last30Days++;
+  }
+  return { total, byType, last7Days, last30Days };
+}
+function verifyChain() {
+  load(true);
+  const hmacProtected = hmacKeyConfigured();
+  if (_blunders.length === 0) {
+    return { valid: true, verified: 0, note: "Log is empty \u2014 nothing to verify", tamperEvidenceLevel: "none", hmacProtected };
+  }
+  const firstV2Idx = _blunders.findIndex((b) => !!b.hash);
+  if (firstV2Idx === -1) {
+    return {
+      valid: true,
+      verified: 0,
+      truncated: false,
+      note: "No cryptographically-verified entries \u2014 all entries predate hash-chain migration. Chain integrity cannot be confirmed.",
+      tamperEvidenceLevel: "none",
+      hmacProtected
+    };
+  }
+  const anchor = _blunders[firstV2Idx];
+  const truncated = anchor.previousHash !== GENESIS_HASH;
+  let expectedPrev = anchor.previousHash;
+  let seenFirstV2 = false;
+  let expectedSeq = null;
+  for (let i = firstV2Idx; i < _blunders.length; i++) {
+    const b = _blunders[i];
+    if (!b.hash) {
+      return {
+        valid: false,
+        firstInvalidIdx: i,
+        reason: `entry at index ${i} (id=${b.id}) is missing hash field after v2 section began \u2014 possible injection`,
+        tamperEvidenceLevel: tamperEvidenceLevel(true),
+        hmacProtected
+      };
+    }
+    if (!seenFirstV2) {
+      seenFirstV2 = true;
+    } else {
+      if (b.previousHash !== expectedPrev) {
+        return {
+          valid: false,
+          firstInvalidIdx: i,
+          reason: `previousHash mismatch at index ${i}: expected ${expectedPrev.slice(0, 8)}\u2026 got ${b.previousHash.slice(0, 8)}\u2026`,
+          tamperEvidenceLevel: tamperEvidenceLevel(true),
+          hmacProtected
+        };
+      }
+    }
+    if (typeof b.sequenceNumber === "number") {
+      if (expectedSeq !== null && b.sequenceNumber !== expectedSeq) {
+        return {
+          valid: false,
+          firstInvalidIdx: i,
+          reason: `sequenceNumber gap at index ${i} (id=${b.id}): expected ${expectedSeq}, got ${b.sequenceNumber} \u2014 an entry may have been removed`,
+          tamperEvidenceLevel: tamperEvidenceLevel(true),
+          hmacProtected
+        };
+      }
+      expectedSeq = b.sequenceNumber + 1;
+    } else {
+      expectedSeq = null;
+    }
+    const { hash: _h, previousHash: _p, ...rest } = b;
+    const recomputed = _computeHash(rest, b.previousHash);
+    if (recomputed !== b.hash) {
+      return {
+        valid: false,
+        firstInvalidIdx: i,
+        reason: `hash mismatch at index ${i} (id=${b.id}): content was modified after recording`,
+        tamperEvidenceLevel: tamperEvidenceLevel(true),
+        hmacProtected
+      };
+    }
+    expectedPrev = b.hash;
+  }
+  const verifiedCount = _blunders.length - firstV2Idx;
+  return {
+    valid: true,
+    truncated,
+    verifiedFrom: anchor.id,
+    verified: verifiedCount,
+    tamperEvidenceLevel: tamperEvidenceLevel(true),
+    hmacProtected
+  };
+}
+function verifyCheckpoints() {
+  load(true);
+  const mismatches = [];
+  let raw;
+  try {
+    raw = fs.readFileSync(CHECKPOINT_FILE, "utf8");
+  } catch {
+    return { checked: 0, mismatches: [] };
+  }
+  const bySeq = /* @__PURE__ */ new Map();
+  for (const b of _blunders) {
+    if (typeof b.sequenceNumber === "number") bySeq.set(b.sequenceNumber, b);
+  }
+  let checked = 0;
+  for (const line of raw.split("\n")) {
+    if (!line.trim()) continue;
+    let checkpoint;
+    try {
+      checkpoint = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    checked++;
+    const current = bySeq.get(checkpoint.sequenceNumber);
+    if (!current) {
+      mismatches.push({
+        sequenceNumber: checkpoint.sequenceNumber,
+        expectedHash: checkpoint.hash,
+        reason: "entry no longer exists in the live chain (evicted by normal ring-buffer rollover, or deleted)"
+      });
+    } else if (current.hash !== checkpoint.hash) {
+      mismatches.push({
+        sequenceNumber: checkpoint.sequenceNumber,
+        expectedHash: checkpoint.hash,
+        reason: `live entry's hash (${current.hash.slice(0, 8)}\u2026) does not match the checkpointed hash \u2014 content was altered after the checkpoint was recorded`
+      });
+    }
+  }
+  return { checked, mismatches };
+}
+export {
+  MAX_BLUNDERS,
+  NON_BLOCKING_BLUNDER_TYPES,
+  _injectRawForTesting,
+  _resetBlunderHmacSecretForTesting,
+  _resetForTesting,
+  getBlunderStats,
+  getBlunders,
+  hmacKeyConfigured,
+  isBlunderIntegrityViolated,
+  isNonBlockingBlunderType,
+  recordBlunder,
+  setBlunderHmacSecret,
+  tamperEvidenceLevel,
+  verifyChain,
+  verifyCheckpoints
+};

package/dist/sensor/agent-context-store.js

@@ -0,0 +1,225 @@
+import initSqlJs from "sql.js";
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { createRequire } from "module";
+import { DATA_DIR } from "./paths.js";
+import logger from "./logger.js";
+import { randomUUID } from "crypto";
+const LEGACY_CONTEXT_DB = path.join(DATA_DIR, "agent-memory.db");
+const CONTEXT_DB = path.join(DATA_DIR, "agent-context.db");
+class AgentContextStore {
+  db = null;
+  resolveWasmPath() {
+    if (process.env.MERGEN_WASM_PATH) return process.env.MERGEN_WASM_PATH;
+    const moduleDir = path.dirname(fileURLToPath(import.meta.url));
+    const fromModule = path.resolve(moduleDir, "../../node_modules/sql.js/dist/sql-wasm.wasm");
+    if (fs.existsSync(fromModule)) return fromModule;
+    try {
+      const req = createRequire(import.meta.url);
+      const resolved = path.join(path.dirname(req.resolve("sql.js")), "sql-wasm.wasm");
+      if (fs.existsSync(resolved)) return resolved;
+    } catch {
+    }
+    return fromModule;
+  }
+  async init() {
+    try {
+      const wasmBinary = fs.readFileSync(this.resolveWasmPath());
+      const SQL = await initSqlJs({ wasmBinary });
+      fs.mkdirSync(DATA_DIR, { recursive: true });
+      if (fs.existsSync(LEGACY_CONTEXT_DB) && !fs.existsSync(CONTEXT_DB)) {
+        try {
+          fs.renameSync(LEGACY_CONTEXT_DB, CONTEXT_DB);
+        } catch {
+        }
+      }
+      let fileBuffer;
+      if (fs.existsSync(CONTEXT_DB)) {
+        try {
+          fileBuffer = fs.readFileSync(CONTEXT_DB);
+        } catch {
+        }
+      }
+      this.db = fileBuffer ? new SQL.Database(fileBuffer) : new SQL.Database();
+      this.db.run(`
+        CREATE TABLE IF NOT EXISTS agent_context (
+          id                TEXT PRIMARY KEY,
+          agent_id          TEXT NOT NULL DEFAULT '',
+          key               TEXT NOT NULL,
+          value             TEXT NOT NULL,
+          stored_at         INTEGER NOT NULL,
+          ttl_ms            INTEGER NOT NULL DEFAULT 0,
+          service           TEXT NOT NULL DEFAULT '',
+          error_fingerprint TEXT NOT NULL DEFAULT ''
+        );
+        CREATE INDEX IF NOT EXISTS idx_agent_context_agent_key ON agent_context (agent_id, key);
+        CREATE INDEX IF NOT EXISTS idx_agent_context_episodic ON agent_context (service, error_fingerprint, stored_at);
+      `);
+      this.flush();
+      try {
+        this.db.run(`ALTER TABLE agent_memory RENAME TO agent_context`);
+      } catch {
+      }
+      try {
+        this.db.run(`ALTER TABLE agent_context ADD COLUMN service TEXT NOT NULL DEFAULT ''`);
+      } catch {
+      }
+      try {
+        this.db.run(`ALTER TABLE agent_context ADD COLUMN error_fingerprint TEXT NOT NULL DEFAULT ''`);
+      } catch {
+      }
+      try {
+        this.db.run(`CREATE INDEX IF NOT EXISTS idx_agent_context_episodic ON agent_context (service, error_fingerprint, stored_at)`);
+      } catch {
+      }
+      logger.debug({ path: CONTEXT_DB }, "agent-context-store: initialized");
+      this.scheduleCleanup();
+    } catch (err) {
+      logger.error({ err }, "agent-context-store: init failed");
+    }
+  }
+  scheduleCleanup() {
+    const run = () => {
+      if (!this.db) return;
+      try {
+        const now = Date.now();
+        this.db.run("DELETE FROM agent_context WHERE ttl_ms > 0 AND stored_at + ttl_ms < ?", [now]);
+        this.flush();
+        try {
+          const stat = fs.statSync(CONTEXT_DB);
+          if (stat.size > 100 * 1024 * 1024) {
+            this.db.run("DELETE FROM agent_context WHERE id IN (SELECT id FROM agent_context ORDER BY stored_at ASC LIMIT 1000)");
+            this.flush();
+            logger.warn("agent-context-store: DB exceeded 100 MB cap \u2014 pruned oldest 1000 entries");
+          }
+        } catch {
+        }
+      } catch (err) {
+        logger.warn({ err }, "agent-context-store: cleanup failed");
+      }
+    };
+    run();
+    setInterval(run, 60 * 6e4).unref();
+  }
+  flush() {
+    if (!this.db) return;
+    try {
+      const data = this.db.export();
+      fs.writeFileSync(CONTEXT_DB, Buffer.from(data));
+    } catch (err) {
+      logger.warn({ err }, "agent-context-store: flush failed");
+    }
+  }
+  store(agentId, key, value, ttlMs = 0, service = "", errorFingerprint = "") {
+    if (!this.db) throw new Error("agent-context-store: not initialized");
+    const now = Date.now();
+    const existing = this.recall(agentId, key, 1);
+    const id = existing[0]?.id ?? randomUUID();
+    this.db.run(
+      `INSERT OR REPLACE INTO agent_context
+         (id, agent_id, key, value, stored_at, ttl_ms, service, error_fingerprint)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+      [id, agentId, key, value, now, ttlMs, service, errorFingerprint]
+    );
+    this.flush();
+    return { id, agentId, key, value, storedAt: now, ttlMs, service, errorFingerprint };
+  }
+  recall(agentId, key, limit = 20, service, errorFingerprint) {
+    if (!this.db) return [];
+    const now = Date.now();
+    const clauses = ["(ttl_ms = 0 OR stored_at + ttl_ms > ?)"];
+    const params = [now];
+    if (agentId) {
+      clauses.push("agent_id = ?");
+      params.push(agentId);
+    }
+    if (key) {
+      clauses.push("key = ?");
+      params.push(key);
+    }
+    if (service) {
+      clauses.push("service = ?");
+      params.push(service);
+    }
+    if (errorFingerprint) {
+      clauses.push("error_fingerprint = ?");
+      params.push(errorFingerprint);
+    }
+    const sql = `SELECT * FROM agent_context WHERE ${clauses.join(" AND ")} ORDER BY stored_at DESC LIMIT ?`;
+    params.push(limit);
+    try {
+      const stmt = this.db.prepare(sql);
+      stmt.bind(params);
+      const rows = [];
+      while (stmt.step()) {
+        const r = stmt.getAsObject();
+        rows.push({
+          id: r["id"],
+          agentId: r["agent_id"],
+          key: r["key"],
+          value: r["value"],
+          storedAt: r["stored_at"],
+          ttlMs: r["ttl_ms"],
+          service: r["service"] ?? "",
+          errorFingerprint: r["error_fingerprint"] ?? ""
+        });
+      }
+      stmt.free();
+      return rows;
+    } catch (err) {
+      logger.warn({ err }, "agent-context-store: recall failed");
+      return [];
+    }
+  }
+  recallEpisodic(service, errorFingerprint, limit = 10) {
+    return this.recall(void 0, void 0, limit, service, errorFingerprint);
+  }
+  isHealthy() {
+    return this.db !== null;
+  }
+  listKeys(agentId) {
+    if (!this.db) return [];
+    const now = Date.now();
+    const clauses = ["(ttl_ms = 0 OR stored_at + ttl_ms > ?)"];
+    const params = [now];
+    if (agentId) {
+      clauses.push("agent_id = ?");
+      params.push(agentId);
+    }
+    const sql = `SELECT agent_id, key, MAX(stored_at) AS last_at
+                 FROM agent_context
+                 WHERE ${clauses.join(" AND ")}
+                 GROUP BY agent_id, key
+                 ORDER BY last_at DESC`;
+    try {
+      const stmt = this.db.prepare(sql);
+      stmt.bind(params);
+      const rows = [];
+      while (stmt.step()) {
+        const r = stmt.getAsObject();
+        rows.push({
+          agentId: r["agent_id"],
+          key: r["key"],
+          lastStoredAt: r["last_at"]
+        });
+      }
+      stmt.free();
+      return rows;
+    } catch (err) {
+      logger.warn({ err }, "agent-context-store: listKeys failed");
+      return [];
+    }
+  }
+  forget(agentId, key) {
+    if (!this.db) return;
+    this.db.run("DELETE FROM agent_context WHERE agent_id = ? AND key = ?", [agentId, key]);
+    this.flush();
+  }
+}
+const agentContextStore = new AgentContextStore();
+const agentMemoryStore = agentContextStore;
+export {
+  agentContextStore,
+  agentMemoryStore
+};