@datasynx/agentic-ai-cartography 2.6.0 → 2.8.0

package/dist/index.cjs

@@ -45,9 +45,11 @@ __export(src_exports, {
   CredentialConfigSchema: () => CredentialConfigSchema,
   CsvCostSource: () => CsvCostSource,
   DEFAULT_ANOMALY_THRESHOLDS: () => DEFAULT_ANOMALY_THRESHOLDS,
+  DEFAULT_INGEST_QUOTA: () => DEFAULT_INGEST_QUOTA,
   DEFAULT_SERVER_NAME: () => DEFAULT_SERVER_NAME,
   DEFAULT_TENANT: () => DEFAULT_TENANT,
   DriftConfigSchema: () => DriftConfigSchema,
+  GraphStoreBackend: () => GraphStoreBackend,
   INGEST_SCHEMA_VERSION: () => INGEST_SCHEMA_VERSION,
   IngestEnvelopeSchema: () => IngestEnvelopeSchema,
   InvalidTenantError: () => InvalidTenantError,
@@ -66,10 +68,12 @@ __export(src_exports, {
   ProviderRegistry: () => ProviderRegistry,
   RELATION_TO_DIRECTION: () => RELATION_TO_DIRECTION,
   ROLES: () => ROLES,
+  RateLimiter: () => RateLimiter,
   RoleSchema: () => RoleSchema,
   RuleCheckSchema: () => RuleCheckSchema,
   RulesetSchema: () => RulesetSchema,
   SCAN_ARG_PATTERNS: () => SCAN_ARG_PATTERNS,
+  SCHEMA_VERSION: () => SCHEMA_VERSION,
   SDL: () => SDL,
   SEVERITY_WEIGHT: () => SEVERITY_WEIGHT,
   SHARING_LEVELS: () => SHARING_LEVELS,
@@ -217,6 +221,7 @@ __export(src_exports, {
   nodesToAssets: () => nodesToAssets,
   normalizeId: () => normalizeId,
   normalizeTenant: () => normalizeTenant,
+  openStoreBackend: () => openStoreBackend,
   orgKeyPath: () => orgKeyPath,
   osUser: () => osUser,
   parseApiArgs: () => parseApiArgs,
@@ -2850,6 +2855,7 @@ function newAnomalies(base, current) {
 
 // src/db.ts
 var DEFAULT_TENANT = "local";
+var SCHEMA_VERSION = 15;
 function normalizeTenant(raw) {
   if (raw == null) return DEFAULT_TENANT;
   const cleaned = sanitizeUntrusted(String(raw)).trim().slice(0, 128);
@@ -4254,6 +4260,14 @@ var CartographyDB = class {
     }
     return rows.length;
   }
+  /**
+   * Retention/compaction (4.7): delete audit events older than `olderThan` (ISO 8601).
+   * The audit trail grows unbounded on a busy collector; this bounds it without touching
+   * sessions/nodes/edges. Returns the number of events removed.
+   */
+  pruneEventsOlderThan(olderThan) {
+    return this.db.prepare("DELETE FROM activity_events WHERE timestamp < ?").run(olderThan).changes;
+  }
   // ── Graph queries (read-only context layer) ─────────────────────────────────
   /** Fetch a single node by id within a session. */
   getNode(sessionId, nodeId) {
@@ -4641,6 +4655,148 @@ var SqliteStoreBackend = class {
   }
 };
 
+// src/store/graph.ts
+function toNum(v) {
+  if (typeof v === "number") return v;
+  if (v && typeof v === "object" && "toNumber" in v && typeof v.toNumber === "function") {
+    return v.toNumber();
+  }
+  return Number(v ?? 0);
+}
+var GraphStoreBackend = class {
+  constructor(driver) {
+    this.driver = driver;
+  }
+  async run(cypher, params) {
+    const session = this.driver.session();
+    try {
+      return await session.run(cypher, params);
+    } finally {
+      await session.close();
+    }
+  }
+  async upsertNode(org, node, identity, contributor) {
+    const res = await this.run(
+      `MERGE (n:Node {org: $org, globalId: $globalId})
+       ON CREATE SET n._created = true
+       ON MATCH  SET n._created = false
+       SET n.id = $id, n.contentHash = $contentHash, n.type = $type, n.name = $name,
+           n.domain = $domain, n.owner = $owner,
+           n.confidence = CASE WHEN n.confidence IS NULL OR $confidence > n.confidence THEN $confidence ELSE n.confidence END
+       MERGE (c:Contributor {org: $org, globalId: $globalId, machineId: $machineId})
+       SET c.hostname = $hostname, c.user = $user, c.at = $at,
+           c.confidence = CASE WHEN c.confidence IS NULL OR $contribConfidence > c.confidence THEN $contribConfidence ELSE c.confidence END
+       RETURN n._created AS created`,
+      {
+        org,
+        globalId: identity.globalId,
+        contentHash: identity.contentHash,
+        id: node.id,
+        type: node.type,
+        name: node.name,
+        domain: node.domain ?? null,
+        owner: node.owner ?? null,
+        confidence: node.confidence,
+        machineId: contributor.machineId,
+        hostname: contributor.hostname,
+        user: contributor.user,
+        at: contributor.at,
+        contribConfidence: contributor.confidence
+      }
+    );
+    return res.records[0]?.get("created") === true ? "created" : "merged";
+  }
+  async insertEdge(org, edge) {
+    await this.run(
+      `MATCH (s:Node {org: $org, id: $source})
+       MATCH (t:Node {org: $org, id: $target})
+       MERGE (s)-[r:DEPENDS {relationship: $rel}]->(t)
+       SET r.evidence = $evidence, r.confidence = $confidence`,
+      { org, source: edge.sourceId, target: edge.targetId, rel: edge.relationship, evidence: edge.evidence, confidence: edge.confidence }
+    );
+  }
+  async getSummary(org) {
+    const totals = await this.run(
+      `MATCH (n:Node {org: $org})
+       OPTIONAL MATCH (n)-[r:DEPENDS]->(:Node {org: $org})
+       RETURN count(DISTINCT n) AS nodes, count(r) AS edges`,
+      { org }
+    );
+    const byType = await this.run(`MATCH (n:Node {org: $org}) RETURN n.type AS k, count(*) AS c`, { org });
+    const byDomain = await this.run(`MATCH (n:Node {org: $org}) RETURN coalesce(n.domain, '(none)') AS k, count(*) AS c`, { org });
+    const byRel = await this.run(`MATCH (:Node {org: $org})-[r:DEPENDS]->(:Node {org: $org}) RETURN r.relationship AS k, count(*) AS c`, { org });
+    const top = await this.run(
+      `MATCH (n:Node {org: $org})
+       OPTIONAL MATCH (n)-[r:DEPENDS]-(:Node {org: $org})
+       RETURN n.id AS id, n.name AS name, n.type AS type, count(r) AS degree
+       ORDER BY degree DESC, id ASC LIMIT 10`,
+      { org }
+    );
+    const contrib = await this.run(`MATCH (c:Contributor {org: $org}) RETURN count(DISTINCT c.machineId) AS contributors`, { org });
+    const counts = (r) => {
+      const out = {};
+      for (const rec of r.records) out[String(rec.get("k"))] = toNum(rec.get("c"));
+      return out;
+    };
+    return {
+      org,
+      totals: { nodes: toNum(totals.records[0]?.get("nodes")), edges: toNum(totals.records[0]?.get("edges")) },
+      nodesByType: counts(byType),
+      nodesByDomain: counts(byDomain),
+      edgesByRelationship: counts(byRel),
+      topConnected: top.records.map((rec) => ({
+        id: String(rec.get("id")),
+        name: String(rec.get("name")),
+        type: String(rec.get("type")),
+        degree: toNum(rec.get("degree"))
+      })),
+      contributors: toNum(contrib.records[0]?.get("contributors"))
+    };
+  }
+  async getContributors(globalId2) {
+    const res = await this.run(
+      `MATCH (c:Contributor {globalId: $globalId})
+       RETURN c.machineId AS machineId, c.hostname AS hostname, c.user AS user, c.org AS org, c.at AS at, c.confidence AS confidence`,
+      { globalId: globalId2 }
+    );
+    return res.records.map((rec) => ({
+      machineId: String(rec.get("machineId")),
+      hostname: String(rec.get("hostname")),
+      user: String(rec.get("user")),
+      organization: rec.get("org") != null ? String(rec.get("org")) : void 0,
+      at: String(rec.get("at")),
+      confidence: toNum(rec.get("confidence"))
+    }));
+  }
+  async close() {
+    await this.driver.close();
+  }
+};
+
+// src/store/index.ts
+async function defaultNeo4jDriver(url, user, password) {
+  const mod = await import("neo4j-driver");
+  return mod.default.driver(url, mod.default.auth.basic(user, password));
+}
+async function openStoreBackend(db, opts = {}) {
+  if (opts.backend === "graph" && opts.graphUrl) {
+    try {
+      const make = opts.driverFactory ?? defaultNeo4jDriver;
+      const driver = await make(opts.graphUrl, opts.graphUser ?? "neo4j", opts.graphPassword ?? "");
+      if (driver.verifyConnectivity) await driver.verifyConnectivity();
+      logInfo("central store: graph backend active", { host: stripSensitive(opts.graphUrl) });
+      return new GraphStoreBackend(driver);
+    } catch (err) {
+      logWarn("central store: graph backend unavailable \u2014 falling back to SQLite", {
+        host: stripSensitive(opts.graphUrl),
+        reason: err instanceof Error ? err.message : String(err)
+      });
+      return new SqliteStoreBackend(db);
+    }
+  }
+  return new SqliteStoreBackend(db);
+}
+
 // src/store/query.ts
 var NotFoundError = class extends Error {
   constructor(message) {
@@ -4945,7 +5101,7 @@ var ContributorSchema = import_zod5.z.object({
 });
 var IngestEnvelopeSchema = import_zod5.z.object({
   schemaVersion: import_zod5.z.literal(INGEST_SCHEMA_VERSION),
-  org: import_zod5.z.string().min(1).optional(),
+  org: import_zod5.z.string().min(1).max(128).optional(),
   items: import_zod5.z.array(import_zod5.z.object({
     contentHash: import_zod5.z.string(),
     kind: import_zod5.z.enum(["node", "edge"]),
@@ -4955,9 +5111,9 @@ var IngestEnvelopeSchema = import_zod5.z.object({
   contributor: ContributorSchema.optional(),
   anonymizationLevel: import_zod5.z.enum(["none", "anonymized", "full"]).optional()
 });
-function ingestEnvelope(store, envelope, opts = {}) {
+async function ingestEnvelope(store, envelope, opts = {}) {
   const anonMode = opts.anonMode ?? "reject";
-  const org = envelope.org ?? opts.defaultOrg ?? "local";
+  const org = normalizeTenant(envelope.org ?? opts.defaultOrg);
   const level = envelope.anonymizationLevel ?? "anonymized";
   const at = (/* @__PURE__ */ new Date()).toISOString();
   const contributor = {
@@ -5008,7 +5164,7 @@ function ingestEnvelope(store, envelope, opts = {}) {
     }
     const safe = check.node;
     const identity = computeIdentity(org, safe);
-    const outcome = store.upsertNode(org, safe, identity, { ...contributor, confidence: safe.confidence });
+    const outcome = await store.upsertNode(org, safe, identity, { ...contributor, confidence: safe.confidence });
     accepted += 1;
     if (outcome === "merged") merged += 1;
     acceptedNodeIds.add(safe.id);
@@ -5024,7 +5180,7 @@ function ingestEnvelope(store, envelope, opts = {}) {
     if (acceptedNodeIds.size > 0 && (!acceptedNodeIds.has(edge.sourceId) || !acceptedNodeIds.has(edge.targetId))) {
       continue;
     }
-    store.insertEdge(org, edge);
+    await store.insertEdge(org, edge);
     edges += 1;
   }
   logInfo("ingest", { org, accepted, merged, rejected, edges, violations, level, anonMode });
@@ -5033,15 +5189,24 @@ function ingestEnvelope(store, envelope, opts = {}) {
 
 // src/central/server.ts
 function createIngestHandler(store, opts = {}) {
-  return (body) => {
+  const quota = opts.quota;
+  return async (body) => {
     const parsed = IngestEnvelopeSchema.safeParse(body);
     if (!parsed.success) {
       const issues = parsed.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`);
       logWarn("ingest: rejected invalid envelope", { issues });
       return { status: 400, body: { error: "invalid envelope", issues } };
     }
+    if (quota) {
+      const org = normalizeTenant(parsed.data.org ?? opts.defaultOrg);
+      const decision = quota.take(org);
+      if (!decision.allowed) {
+        logWarn("ingest: rate limited", { org, retryAfterSec: decision.retryAfterSec });
+        return { status: 429, body: { error: "too many requests" }, headers: { "retry-after": String(decision.retryAfterSec) } };
+      }
+    }
     try {
-      const result = ingestEnvelope(store, parsed.data, opts);
+      const result = await ingestEnvelope(store, parsed.data, opts);
       return { status: 200, body: result };
     } catch (err) {
       logWarn("ingest: failed", { error: err instanceof Error ? err.message : String(err) });
@@ -5050,6 +5215,39 @@ function createIngestHandler(store, opts = {}) {
   };
 }
 
+// src/central/quota.ts
+var DEFAULT_INGEST_QUOTA = { capacity: 120, refillMs: 6e4 };
+var MAX_KEYS = 1e4;
+var RateLimiter = class {
+  constructor(cfg = DEFAULT_INGEST_QUOTA, now = () => Date.now()) {
+    this.cfg = cfg;
+    this.now = now;
+  }
+  buckets = /* @__PURE__ */ new Map();
+  /** Consume one token for `key`. Returns whether the request is allowed (+ Retry-After when not). */
+  take(key) {
+    const t = this.now();
+    const ratePerMs = this.cfg.capacity / this.cfg.refillMs;
+    let b = this.buckets.get(key);
+    if (!b) {
+      if (this.buckets.size >= MAX_KEYS) {
+        const oldest = this.buckets.keys().next().value;
+        if (oldest !== void 0) this.buckets.delete(oldest);
+      }
+      b = { tokens: this.cfg.capacity, last: t };
+      this.buckets.set(key, b);
+    }
+    b.tokens = Math.min(this.cfg.capacity, b.tokens + Math.max(0, t - b.last) * ratePerMs);
+    b.last = t;
+    if (b.tokens >= 1) {
+      b.tokens -= 1;
+      return { allowed: true, retryAfterSec: 0 };
+    }
+    const waitMs = (1 - b.tokens) / ratePerMs;
+    return { allowed: false, retryAfterSec: Math.max(1, Math.ceil(waitMs / 1e3)) };
+  }
+};
+
 // src/scanners/bookmarks.ts
 var PERSONAL = [
   "facebook.",
@@ -5937,7 +6135,7 @@ async function resolveNlQuery(db, sessionId, search, raw, opts) {
 
 // src/mcp/server.ts
 var SERVER_NAME = "cartography";
-var SERVER_VERSION = "2.6.0";
+var SERVER_VERSION = "2.8.0";
 var SERVICE_TYPES = NODE_TYPE_GROUPS.web;
 var DATA_TYPES = NODE_TYPE_GROUPS.data;
 var lexicalSearch = async (db, sessionId, query, opts) => db.searchNodes(sessionId, query, { types: opts.types, limit: opts.limit }).map((node) => ({ node }));
@@ -6012,9 +6210,10 @@ function createMcpServer(opts = {}) {
     "graph-summary",
     "cartography://graph/summary",
     { title: "Topology summary", description: "Low-token aggregate index of the whole landscape \u2014 read this first.", mimeType: "text/markdown" },
-    (uri) => {
+    async (uri) => {
       if (org !== void 0) {
-        return { contents: [{ uri: uri.href, mimeType: "text/markdown", text: summaryText(db.getOrgSummary(org)) }] };
+        const s = opts.orgSummary ? await opts.orgSummary(org) : db.getOrgSummary(org);
+        return { contents: [{ uri: uri.href, mimeType: "text/markdown", text: summaryText(s) }] };
       }
       const sid = resolveSession();
       if (!sid) return { contents: [{ uri: uri.href, mimeType: "text/markdown", text: "No discovery session found. Run discovery first." }] };
@@ -6089,8 +6288,8 @@ function createMcpServer(opts = {}) {
   server.registerTool(
     "get_summary",
     { title: "Get topology summary", description: "Low-token overview of the whole landscape (counts, types, domains, most-connected, anomalies).", inputSchema: {}, annotations: readOnly },
-    () => {
-      if (org !== void 0) return json(db.getOrgSummary(org));
+    async () => {
+      if (org !== void 0) return json(opts.orgSummary ? await opts.orgSummary(org) : db.getOrgSummary(org));
       const sid = resolveSession();
       if (!sid) return json({ error: "No discovery session found." });
       return json(db.getGraphSummary(sid));
@@ -6580,6 +6779,16 @@ async function runHttp(factory, opts = {}) {
   const httpServer = import_node_http.default.createServer(async (req, res) => {
     try {
       const url = req.url ?? "";
+      const probePath = new URL(url || "/", "http://probe").pathname;
+      if (probePath === "/healthz") {
+        res.writeHead(200, { "content-type": "application/json" }).end('{"status":"ok"}');
+        return;
+      }
+      if (probePath === "/readyz") {
+        const r = opts.readiness ? opts.readiness() : { ready: true };
+        res.writeHead(r.ready ? 200 : 503, { "content-type": "application/json" }).end(JSON.stringify({ status: r.ready ? "ready" : "unready" }));
+        return;
+      }
       const isIngest = url.startsWith("/ingest") && opts.onIngest !== void 0;
       if (!url.startsWith("/mcp") && !isIngest) {
         res.writeHead(404, { "content-type": "application/json" }).end('{"error":"not found"}');
@@ -6606,8 +6815,8 @@ async function runHttp(factory, opts = {}) {
           res.writeHead(413, { "content-type": "application/json" }).end('{"error":"payload too large"}');
           return;
         }
-        const out = onIngest(value);
-        res.writeHead(out.status, { "content-type": "application/json" }).end(JSON.stringify(out.body));
+        const out = await onIngest(value);
+        res.writeHead(out.status, { "content-type": "application/json", ...out.headers ?? {} }).end(JSON.stringify(out.body));
         return;
       }
       const sessionId = req.headers["mcp-session-id"];
@@ -11859,9 +12068,11 @@ function checkClaudePrerequisites() {
   CredentialConfigSchema,
   CsvCostSource,
   DEFAULT_ANOMALY_THRESHOLDS,
+  DEFAULT_INGEST_QUOTA,
   DEFAULT_SERVER_NAME,
   DEFAULT_TENANT,
   DriftConfigSchema,
+  GraphStoreBackend,
   INGEST_SCHEMA_VERSION,
   IngestEnvelopeSchema,
   InvalidTenantError,
@@ -11880,10 +12091,12 @@ function checkClaudePrerequisites() {
   ProviderRegistry,
   RELATION_TO_DIRECTION,
   ROLES,
+  RateLimiter,
   RoleSchema,
   RuleCheckSchema,
   RulesetSchema,
   SCAN_ARG_PATTERNS,
+  SCHEMA_VERSION,
   SDL,
   SEVERITY_WEIGHT,
   SHARING_LEVELS,
@@ -12031,6 +12244,7 @@ function checkClaudePrerequisites() {
   nodesToAssets,
   normalizeId,
   normalizeTenant,
+  openStoreBackend,
   orgKeyPath,
   osUser,
   parseApiArgs,