mergen-server 1.4.0

package/dist/sensor/agent-memory-store.js

@@ -0,0 +1,231 @@
+import initSqlJs from "sql.js";
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { createRequire } from "module";
+import { DATA_DIR } from "./paths.js";
+import logger from "./logger.js";
+import { randomUUID } from "crypto";
+const MEMORY_DB = path.join(DATA_DIR, "agent-memory.db");
+class AgentMemoryStore {
+  db = null;
+  resolveWasmPath() {
+    if (process.env.MERGEN_WASM_PATH) return process.env.MERGEN_WASM_PATH;
+    const moduleDir = path.dirname(fileURLToPath(import.meta.url));
+    const fromModule = path.resolve(moduleDir, "../../node_modules/sql.js/dist/sql-wasm.wasm");
+    if (fs.existsSync(fromModule)) return fromModule;
+    try {
+      const req = createRequire(import.meta.url);
+      const resolved = path.join(path.dirname(req.resolve("sql.js")), "sql-wasm.wasm");
+      if (fs.existsSync(resolved)) return resolved;
+    } catch {
+    }
+    return fromModule;
+  }
+  async init() {
+    try {
+      const wasmBinary = fs.readFileSync(this.resolveWasmPath());
+      const SQL = await initSqlJs({ wasmBinary });
+      fs.mkdirSync(DATA_DIR, { recursive: true });
+      let fileBuffer;
+      if (fs.existsSync(MEMORY_DB)) {
+        try {
+          fileBuffer = fs.readFileSync(MEMORY_DB);
+        } catch {
+        }
+      }
+      this.db = fileBuffer ? new SQL.Database(fileBuffer) : new SQL.Database();
+      this.db.run(`
+        CREATE TABLE IF NOT EXISTS agent_memory (
+          id                TEXT PRIMARY KEY,
+          agent_id          TEXT NOT NULL DEFAULT '',
+          key               TEXT NOT NULL,
+          value             TEXT NOT NULL,
+          stored_at         INTEGER NOT NULL,
+          ttl_ms            INTEGER NOT NULL DEFAULT 0,
+          service           TEXT NOT NULL DEFAULT '',
+          error_fingerprint TEXT NOT NULL DEFAULT ''
+        );
+        CREATE INDEX IF NOT EXISTS idx_agent_memory_agent_key ON agent_memory (agent_id, key);
+        CREATE INDEX IF NOT EXISTS idx_agent_memory_episodic ON agent_memory (service, error_fingerprint, stored_at);
+      `);
+      this.flush();
+      try {
+        this.db.run(`ALTER TABLE agent_memory ADD COLUMN service TEXT NOT NULL DEFAULT ''`);
+      } catch {
+      }
+      try {
+        this.db.run(`ALTER TABLE agent_memory ADD COLUMN error_fingerprint TEXT NOT NULL DEFAULT ''`);
+      } catch {
+      }
+      try {
+        this.db.run(`CREATE INDEX IF NOT EXISTS idx_agent_memory_episodic ON agent_memory (service, error_fingerprint, stored_at)`);
+      } catch {
+      }
+      logger.debug({ path: MEMORY_DB }, "agent-memory-store: initialized");
+      this.scheduleCleanup();
+    } catch (err) {
+      logger.error({ err }, "agent-memory-store: init failed");
+    }
+  }
+  scheduleCleanup() {
+    const run = () => {
+      if (!this.db) return;
+      try {
+        const now = Date.now();
+        this.db.run("DELETE FROM agent_memory WHERE ttl_ms > 0 AND stored_at + ttl_ms < ?", [now]);
+        this.flush();
+        try {
+          const stat = fs.statSync(MEMORY_DB);
+          if (stat.size > 100 * 1024 * 1024) {
+            this.db.run("DELETE FROM agent_memory WHERE id IN (SELECT id FROM agent_memory ORDER BY stored_at ASC LIMIT 1000)");
+            this.flush();
+            logger.warn("agent-memory-store: DB exceeded 100 MB cap \u2014 pruned oldest 1000 entries");
+          }
+        } catch {
+        }
+      } catch (err) {
+        logger.warn({ err }, "agent-memory-store: cleanup failed");
+      }
+    };
+    run();
+    setInterval(run, 60 * 6e4).unref();
+  }
+  flush() {
+    if (!this.db) return;
+    try {
+      const data = this.db.export();
+      fs.writeFileSync(MEMORY_DB, Buffer.from(data));
+    } catch (err) {
+      logger.warn({ err }, "agent-memory-store: flush failed");
+    }
+  }
+  /**
+   * Store or overwrite a memory entry.
+   * If an entry with the same (agentId, key) already exists it is replaced.
+   * Optional `service` and `errorFingerprint` enable episodic recall.
+   */
+  store(agentId, key, value, ttlMs = 0, service = "", errorFingerprint = "") {
+    if (!this.db) throw new Error("agent-memory-store: not initialized");
+    const now = Date.now();
+    const existing = this.recall(agentId, key, 1);
+    const id = existing[0]?.id ?? randomUUID();
+    this.db.run(
+      `INSERT OR REPLACE INTO agent_memory
+         (id, agent_id, key, value, stored_at, ttl_ms, service, error_fingerprint)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+      [id, agentId, key, value, now, ttlMs, service, errorFingerprint]
+    );
+    this.flush();
+    return { id, agentId, key, value, storedAt: now, ttlMs, service, errorFingerprint };
+  }
+  /**
+   * Retrieve memory entries filtered by agentId, key, service, and/or errorFingerprint.
+   * Expired entries are excluded. Results are ordered newest-first.
+   */
+  recall(agentId, key, limit = 20, service, errorFingerprint) {
+    if (!this.db) return [];
+    const now = Date.now();
+    const clauses = ["(ttl_ms = 0 OR stored_at + ttl_ms > ?)"];
+    const params = [now];
+    if (agentId) {
+      clauses.push("agent_id = ?");
+      params.push(agentId);
+    }
+    if (key) {
+      clauses.push("key = ?");
+      params.push(key);
+    }
+    if (service) {
+      clauses.push("service = ?");
+      params.push(service);
+    }
+    if (errorFingerprint) {
+      clauses.push("error_fingerprint = ?");
+      params.push(errorFingerprint);
+    }
+    const sql = `SELECT * FROM agent_memory WHERE ${clauses.join(" AND ")} ORDER BY stored_at DESC LIMIT ?`;
+    params.push(limit);
+    try {
+      const stmt = this.db.prepare(sql);
+      stmt.bind(params);
+      const rows = [];
+      while (stmt.step()) {
+        const r = stmt.getAsObject();
+        rows.push({
+          id: r["id"],
+          agentId: r["agent_id"],
+          key: r["key"],
+          value: r["value"],
+          storedAt: r["stored_at"],
+          ttlMs: r["ttl_ms"],
+          service: r["service"] ?? "",
+          errorFingerprint: r["error_fingerprint"] ?? ""
+        });
+      }
+      stmt.free();
+      return rows;
+    } catch (err) {
+      logger.warn({ err }, "agent-memory-store: recall failed");
+      return [];
+    }
+  }
+  /**
+   * Episodic recall: return memories for a specific (service, errorFingerprint) context,
+   * ordered by recency. This is the associative lookup LeCun-style models use.
+   */
+  recallEpisodic(service, errorFingerprint, limit = 10) {
+    return this.recall(void 0, void 0, limit, service, errorFingerprint);
+  }
+  /** Returns true when the SQLite database initialised successfully. */
+  isHealthy() {
+    return this.db !== null;
+  }
+  /**
+   * List all non-expired keys stored by an agent, grouped for discovery.
+   * Lets an agent enumerate what it has stored without knowing keys in advance.
+   */
+  listKeys(agentId) {
+    if (!this.db) return [];
+    const now = Date.now();
+    const clauses = ["(ttl_ms = 0 OR stored_at + ttl_ms > ?)"];
+    const params = [now];
+    if (agentId) {
+      clauses.push("agent_id = ?");
+      params.push(agentId);
+    }
+    const sql = `SELECT agent_id, key, MAX(stored_at) AS last_at
+                 FROM agent_memory
+                 WHERE ${clauses.join(" AND ")}
+                 GROUP BY agent_id, key
+                 ORDER BY last_at DESC`;
+    try {
+      const stmt = this.db.prepare(sql);
+      stmt.bind(params);
+      const rows = [];
+      while (stmt.step()) {
+        const r = stmt.getAsObject();
+        rows.push({
+          agentId: r["agent_id"],
+          key: r["key"],
+          lastStoredAt: r["last_at"]
+        });
+      }
+      stmt.free();
+      return rows;
+    } catch (err) {
+      logger.warn({ err }, "agent-memory-store: listKeys failed");
+      return [];
+    }
+  }
+  /** Remove a specific memory by (agentId, key). */
+  forget(agentId, key) {
+    if (!this.db) return;
+    this.db.run("DELETE FROM agent_memory WHERE agent_id = ? AND key = ?", [agentId, key]);
+    this.flush();
+  }
+}
+const agentMemoryStore = new AgentMemoryStore();
+export {
+  agentMemoryStore
+};

package/dist/sensor/audit-fetch.js

@@ -0,0 +1,23 @@
+import { getStores } from "../storage/store-registry.js";
+import { getAuditLog } from "./audit-log.js";
+async function fetchBlunderEntries(from, to) {
+  const blunders = await getStores().blunders.list();
+  return blunders.filter((b) => b.recordedAt >= from && b.recordedAt <= to);
+}
+async function fetchHttpAuditEntries(from, to, limit = 5e4) {
+  return getAuditLog(limit).filter((e) => {
+    const ts = new Date(e.ts).getTime();
+    return ts >= from && ts <= to;
+  });
+}
+async function fetchChainVerification() {
+  const blunderStore = getStores().blunders;
+  const hasEntries = (await blunderStore.list()).length > 0;
+  if (!hasEntries) return { valid: true, verified: 0 };
+  return blunderStore.verifyChain();
+}
+export {
+  fetchBlunderEntries,
+  fetchChainVerification,
+  fetchHttpAuditEntries
+};

package/dist/sensor/audit-log.js

@@ -0,0 +1,273 @@
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { createRequire } from "module";
+import initSqlJs from "sql.js";
+import { AUDIT_LOG, AUDIT_DB, DATA_DIR } from "./paths.js";
+import logger from "./logger.js";
+const MAX_AUDIT_BYTES = 10 * 1024 * 1024;
+const SKIP_PATHS = /* @__PURE__ */ new Set(["/", "/health", "/metrics", "/dashboard", "/local-secret"]);
+class ComplianceLedgerStore {
+  db = null;
+  resolveWasmPath() {
+    if (process.env.MERGEN_WASM_PATH) return process.env.MERGEN_WASM_PATH;
+    const moduleDir = path.dirname(fileURLToPath(import.meta.url));
+    const fromModule = path.resolve(moduleDir, "../../node_modules/sql.js/dist/sql-wasm.wasm");
+    if (fs.existsSync(fromModule)) return fromModule;
+    try {
+      const req = createRequire(import.meta.url);
+      const resolved = path.join(path.dirname(req.resolve("sql.js")), "sql-wasm.wasm");
+      if (fs.existsSync(resolved)) return resolved;
+    } catch {
+    }
+    return fromModule;
+  }
+  async init() {
+    try {
+      const wasmBinary = fs.readFileSync(this.resolveWasmPath());
+      const SQL = await initSqlJs({ wasmBinary });
+      fs.mkdirSync(DATA_DIR, { recursive: true });
+      let fileBuffer;
+      if (fs.existsSync(AUDIT_DB)) {
+        try {
+          fileBuffer = fs.readFileSync(AUDIT_DB);
+        } catch {
+        }
+      }
+      this.db = fileBuffer ? new SQL.Database(fileBuffer) : new SQL.Database();
+      this.db.run(`
+        CREATE TABLE IF NOT EXISTS http_audit (
+          id          INTEGER PRIMARY KEY AUTOINCREMENT,
+          ts          TEXT NOT NULL,
+          actor       TEXT NOT NULL,
+          method      TEXT NOT NULL,
+          path        TEXT NOT NULL,
+          status      INTEGER NOT NULL,
+          duration_ms INTEGER NOT NULL,
+          ip          TEXT NOT NULL
+        );
+        CREATE TABLE IF NOT EXISTS execution_audit (
+          id           INTEGER PRIMARY KEY AUTOINCREMENT,
+          ts           TEXT NOT NULL,
+          event        TEXT NOT NULL,
+          actor        TEXT NOT NULL,
+          cmd          TEXT NOT NULL,
+          ok           INTEGER NOT NULL,
+          exit_code    INTEGER NOT NULL,
+          duration_ms  INTEGER NOT NULL,
+          blocked      INTEGER NOT NULL,
+          block_reason TEXT NOT NULL,
+          timed_out    INTEGER NOT NULL
+        );
+        CREATE INDEX IF NOT EXISTS idx_http_audit_ts ON http_audit (ts);
+        CREATE INDEX IF NOT EXISTS idx_execution_audit_ts ON execution_audit (ts);
+      `);
+      this.flush();
+      logger.info({ path: AUDIT_DB }, "Compliance ledger SQLite store initialised");
+    } catch (err) {
+      logger.error({ err }, "Compliance ledger store failed to initialize");
+    }
+  }
+  flush() {
+    if (!this.db) return;
+    try {
+      const data = this.db.export();
+      fs.writeFileSync(AUDIT_DB, Buffer.from(data));
+    } catch (err) {
+      logger.warn({ err }, "Compliance ledger store flush failed");
+    }
+  }
+  insertHttp(entry) {
+    if (!this.db) return;
+    try {
+      this.db.run(
+        `INSERT INTO http_audit (ts, actor, method, path, status, duration_ms, ip) VALUES (?, ?, ?, ?, ?, ?, ?)`,
+        [entry.ts, entry.actor, entry.method, entry.path, entry.status, entry.durationMs, entry.ip]
+      );
+      this.flush();
+    } catch (err) {
+      logger.warn({ err }, "Failed to insert HTTP audit record");
+    }
+  }
+  insertExecution(entry) {
+    if (!this.db) return;
+    try {
+      this.db.run(
+        `INSERT INTO execution_audit (ts, event, actor, cmd, ok, exit_code, duration_ms, blocked, block_reason, timed_out)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        [
+          entry.ts,
+          entry.event,
+          entry.actor,
+          entry.cmd,
+          entry.ok ? 1 : 0,
+          entry.exitCode,
+          entry.durationMs,
+          entry.blocked ? 1 : 0,
+          entry.blockReason,
+          entry.timedOut ? 1 : 0
+        ]
+      );
+      this.flush();
+    } catch (err) {
+      logger.warn({ err }, "Failed to insert execution audit record");
+    }
+  }
+  getHttpEntries(limit = 200) {
+    if (!this.db) return [];
+    try {
+      const res = this.db.exec(`SELECT ts, actor, method, path, status, duration_ms, ip FROM http_audit ORDER BY ts DESC LIMIT ?`, [limit]);
+      if (res.length === 0) return [];
+      return res[0].values.map((row) => ({
+        ts: String(row[0]),
+        actor: String(row[1]),
+        method: String(row[2]),
+        path: String(row[3]),
+        status: Number(row[4]),
+        durationMs: Number(row[5]),
+        ip: String(row[6])
+      }));
+    } catch (err) {
+      logger.warn({ err }, "Failed to query HTTP audit entries");
+      return [];
+    }
+  }
+  getExecutionEntries(limit = 200) {
+    if (!this.db) return [];
+    try {
+      const res = this.db.exec(
+        `SELECT ts, event, actor, cmd, ok, exit_code, duration_ms, blocked, block_reason, timed_out
+         FROM execution_audit ORDER BY ts DESC LIMIT ?`,
+        [limit]
+      );
+      if (res.length === 0) return [];
+      return res[0].values.map((row) => ({
+        ts: String(row[0]),
+        event: String(row[1]),
+        actor: String(row[2]),
+        cmd: String(row[3]),
+        ok: Number(row[4]) === 1,
+        exitCode: Number(row[5]),
+        durationMs: Number(row[6]),
+        blocked: Number(row[7]) === 1,
+        blockReason: String(row[8]),
+        timedOut: Number(row[9]) === 1
+      }));
+    } catch (err) {
+      logger.warn({ err }, "Failed to query execution audit entries");
+      return [];
+    }
+  }
+}
+const complianceLedgerStore = new ComplianceLedgerStore();
+function resolveActor(req) {
+  const member = req.headers["x-mergen-member"];
+  if (member) return member;
+  return req.ip ?? req.socket.remoteAddress ?? "unknown";
+}
+let _writeQueue = Promise.resolve();
+let _lastAuditWriteError = null;
+let _auditWriteOk = true;
+function getAuditHealth() {
+  return { ok: _auditWriteOk, lastError: _lastAuditWriteError };
+}
+async function appendEntryAsync(entry) {
+  try {
+    await fs.promises.mkdir(DATA_DIR, { recursive: true });
+    try {
+      const stat = await fs.promises.stat(AUDIT_LOG);
+      if (stat.size >= MAX_AUDIT_BYTES) {
+        const rotated = AUDIT_LOG + ".1";
+        try {
+          await fs.promises.unlink(rotated);
+        } catch {
+        }
+        await fs.promises.rename(AUDIT_LOG, rotated);
+      }
+    } catch {
+    }
+    await fs.promises.appendFile(AUDIT_LOG, JSON.stringify(entry) + "\n", "utf8");
+    _auditWriteOk = true;
+    _lastAuditWriteError = null;
+  } catch (err) {
+    _auditWriteOk = false;
+    _lastAuditWriteError = err instanceof Error ? err.message : String(err);
+    logger.error({ err }, "audit-log: write failed \u2014 record dropped");
+  }
+}
+function appendEntry(entry) {
+  complianceLedgerStore.insertHttp(entry);
+  _writeQueue = _writeQueue.then(() => appendEntryAsync(entry)).catch((err) => {
+    _auditWriteOk = false;
+    _lastAuditWriteError = err instanceof Error ? err.message : String(err);
+    logger.error({ err }, "audit-log: write queue error \u2014 record dropped");
+  });
+}
+function auditMiddleware(req, res, next) {
+  if (SKIP_PATHS.has(req.path) || req.path.startsWith("/dashboard")) {
+    next();
+    return;
+  }
+  const start = Date.now();
+  res.on("finish", () => {
+    appendEntry({
+      ts: (/* @__PURE__ */ new Date()).toISOString(),
+      actor: resolveActor(req),
+      method: req.method,
+      path: req.path,
+      status: res.statusCode,
+      durationMs: Date.now() - start,
+      ip: req.ip ?? req.socket.remoteAddress ?? "unknown"
+    });
+  });
+  next();
+}
+function getAuditLog(limit = 200) {
+  const sqliteEntries = complianceLedgerStore.getHttpEntries(limit);
+  if (sqliteEntries.length > 0) return sqliteEntries;
+  try {
+    const raw = fs.readFileSync(AUDIT_LOG, "utf8");
+    const lines = raw.trim().split("\n").filter(Boolean);
+    return lines.slice(-Math.min(limit, 2e3)).map((l) => JSON.parse(l)).reverse();
+  } catch {
+    return [];
+  }
+}
+function recordExecutionAudit(entry) {
+  complianceLedgerStore.insertExecution(entry);
+  const fileEntry = JSON.stringify({
+    t: entry.ts,
+    event: entry.event,
+    actor: entry.actor,
+    cmd: entry.cmd,
+    ok: entry.ok,
+    exitCode: entry.exitCode,
+    durationMs: entry.durationMs,
+    blocked: entry.blocked,
+    blockReason: entry.blockReason,
+    timedOut: entry.timedOut
+  });
+  _writeQueue = _writeQueue.then(async () => {
+    try {
+      await fs.promises.mkdir(DATA_DIR, { recursive: true });
+      await fs.promises.appendFile(AUDIT_LOG, fileEntry + "\n", "utf8");
+      _auditWriteOk = true;
+      _lastAuditWriteError = null;
+    } catch (err) {
+      _auditWriteOk = false;
+      _lastAuditWriteError = err instanceof Error ? err.message : String(err);
+      logger.error({ err }, "audit-log: execution record write failed \u2014 record dropped");
+    }
+  }).catch((err) => {
+    _auditWriteOk = false;
+    _lastAuditWriteError = err instanceof Error ? err.message : String(err);
+    logger.error({ err }, "audit-log: write queue error on execution record");
+  });
+}
+export {
+  auditMiddleware,
+  complianceLedgerStore,
+  getAuditHealth,
+  getAuditLog,
+  recordExecutionAudit
+};