@cohortapp/agent-sdk 2.3.0

package/scripts/daemon/context-compiler.mjs

@@ -0,0 +1,605 @@
+#!/usr/bin/env node
+/**
+ * context-compiler.mjs — Pre-compiles all relevant context for a daemon-spawned session.
+ *
+ * Single export: compileContext(item, classResult, options)
+ *
+ * Assembles sender profile, disclosure boundaries, conversation history,
+ * active sessions, and sent-message registry into a single context block.
+ * Filters history entries by information barrier domains.
+ *
+ * Adaptive token budget: 12K for critical/high, 8K for normal, 4K for low.
+ */
+
+import { readFileSync, readdirSync } from "fs";
+import { join } from "path";
+import { isEnabled as orgEnabled, loadOrgConfig } from "../../lib/org/client.mjs";
+import { recall as orgRecall } from "../../lib/org/knowledge.mjs";
+
+const AGENT_REPO_DIR = process.env.__TEST_AGENT_DIR || process.env.AGENT_DIR || join(new URL(".", import.meta.url).pathname, "../..");
+const SESSION_DIR = process.env.__TEST_SESSION_DIR || join(AGENT_REPO_DIR, "state", "sessions");
+const CHARS_PER_TOKEN = 4;
+
+// ── Org shared-memory (central store via knowledge.* RPC) ───────────
+// The compiled context draws not only from local files but from the org's
+// shared, ACL'd memory plane. Bounded so a large recall can never blow up the
+// prompt: at most this many facts, each truncated. The SERVER enforces ACL —
+// we render exactly what recall() returns.
+const ORG_RECALL_MAX_FACTS = 6;
+const ORG_RECALL_FACT_CHARS = 400;
+
+// ── Token budgets by priority ──────────────────────────────────────
+
+const TOKEN_BUDGETS = {
+  critical: 12000,
+  high:     12000,
+  normal:   8000,
+  low:      4000,
+};
+
+// Fixed allocations (in tokens)
+const ALLOC = {
+  profile:         500,
+  boundaries:      300,
+  active_sessions: 300,
+  sent_registry:   500,
+  org_memory:      600,
+};
+
+// History gets 60% of remaining budget
+const HISTORY_SHARE = 0.6;
+
+// ── Helpers ────────────────────────────────────────────────────────
+
+/**
+ * Extract Slack channel ID from item metadata.
+ */
+function extractChannelId(item) {
+  if (!item) return null;
+  if (item.channel_id) return item.channel_id;
+  if (item.raw_ref) {
+    const match = item.raw_ref.match(/slack:([DC][A-Z0-9]+):/);
+    if (match) return match[1];
+  }
+  if (item.channel && /^[DC][A-Z0-9]{8,}$/.test(item.channel)) return item.channel;
+  return null;
+}
+
+/**
+ * Read a YAML file and truncate to a character budget. Returns null if missing.
+ */
+function loadUserProfile(senderSlug, budgetTokens) {
+  if (!senderSlug) return null;
+  try {
+    const filePath = join(AGENT_REPO_DIR, "memory", "profiles", "users", `${senderSlug}.yaml`);
+    let raw = readFileSync(filePath, "utf-8");
+    const charBudget = budgetTokens * CHARS_PER_TOKEN;
+    if (raw.length > charBudget) {
+      raw = raw.substring(0, charBudget) + "\n...(truncated)";
+    }
+    return raw;
+  } catch {
+    return null;
+  }
+}
+
+// ── Domain / disclosure helpers ────────────────────────────────────
+
+/**
+ * Parse domain descriptions from information-barriers.yaml using regex.
+ * Returns a Map of domain -> description.
+ */
+function loadDomainDescriptions() {
+  try {
+    const filePath = join(AGENT_REPO_DIR, "policies", "information-barriers.yaml");
+    const raw = readFileSync(filePath, "utf-8");
+    const map = new Map();
+    const regex = /^\s{2}([\w-]+):\s*\n\s+description:\s*"(.+?)"/gm;
+    let match;
+    while ((match = regex.exec(raw)) !== null) {
+      map.set(match[1], match[2]);
+    }
+    return map;
+  } catch {
+    return new Map();
+  }
+}
+
+/**
+ * Get all domain names except "public".
+ */
+function getAllDomains() {
+  const descriptions = loadDomainDescriptions();
+  return [...descriptions.keys()].filter((d) => d !== "public");
+}
+
+// Common words to exclude from keyword matching (4+ chars)
+const COMMON_WORDS = new Set([
+  "this", "that", "with", "from", "have", "been", "will", "more", "when",
+  "were", "some", "them", "than", "each", "make", "like", "over", "such",
+  "take", "only", "come", "into", "also", "back", "after", "would", "what",
+  "your", "about", "could", "their", "which", "other", "there", "first",
+  "does", "these", "should", "through", "where", "being", "those",
+  "between", "before", "under", "still",
+]);
+
+/**
+ * Build a map of domain -> keyword array from domain descriptions.
+ * Words must be 4+ chars and not in the common-words exclusion list.
+ */
+function buildDomainKeywords() {
+  const descriptions = loadDomainDescriptions();
+  const result = new Map();
+  for (const [domain, desc] of descriptions) {
+    if (domain === "public") continue;
+    const words = desc
+      .toLowerCase()
+      .replace(/[^a-z0-9\s-]/g, "")
+      .split(/\s+/)
+      .filter((w) => w.length >= 4 && !COMMON_WORDS.has(w));
+    // Deduplicate
+    result.set(domain, [...new Set(words)]);
+  }
+  return result;
+}
+
+/**
+ * Returns true if the text matches 2+ keywords from any forbidden domain.
+ */
+function matchesForbiddenDomain(text, forbiddenDomains, domainKeywords) {
+  if (!forbiddenDomains || forbiddenDomains.length === 0) return false;
+  if (!text) return false;
+  const lower = text.toLowerCase();
+  for (const domain of forbiddenDomains) {
+    const keywords = domainKeywords.get(domain);
+    if (!keywords) continue;
+    let hits = 0;
+    for (const kw of keywords) {
+      if (lower.includes(kw)) {
+        hits++;
+        if (hits >= 2) return true;
+      }
+    }
+  }
+  return false;
+}
+
+/**
+ * Load disclosure boundaries for a sender.
+ * Returns { boundariesSection, forbiddenDomains }.
+ */
+function loadDisclosureBoundaries(senderSlug, budgetTokens) {
+  // Attempt to load profile
+  let privilegeLevel = null;
+  let excludedDomains = [];
+
+  if (senderSlug) {
+    try {
+      const filePath = join(AGENT_REPO_DIR, "memory", "profiles", "users", `${senderSlug}.yaml`);
+      const raw = readFileSync(filePath, "utf-8");
+
+      // Extract privilege_level via regex
+      const privMatch = raw.match(/privilege_level:\s*(\S+)/);
+      if (privMatch) privilegeLevel = privMatch[1];
+
+      // Extract excluded_domains list
+      const exclMatch = raw.match(/excluded_domains:\s*\n((?:\s+-\s+\S+\n?)*)/);
+      if (exclMatch) {
+        excludedDomains = exclMatch[1]
+          .split("\n")
+          .map((line) => line.replace(/^\s*-\s*/, "").trim())
+          .filter(Boolean);
+      }
+    } catch {
+      // No profile — unknown sender
+    }
+  }
+
+  // CEO gets no restrictions
+  if (privilegeLevel === "ceo") {
+    return { boundariesSection: null, forbiddenDomains: [] };
+  }
+
+  // No profile or unknown → restrict to "public" only (forbid all other domains)
+  let forbiddenDomains;
+  if (!privilegeLevel) {
+    forbiddenDomains = getAllDomains();
+  } else {
+    // Has profile with explicit excluded_domains
+    forbiddenDomains = excludedDomains.length > 0 ? excludedDomains : [];
+  }
+
+  if (forbiddenDomains.length === 0) {
+    return { boundariesSection: null, forbiddenDomains: [] };
+  }
+
+  // Build human-readable section
+  const descriptions = loadDomainDescriptions();
+  const lines = ["DO NOT mention or disclose information related to:"];
+  for (const domain of forbiddenDomains) {
+    const desc = descriptions.get(domain) || domain;
+    lines.push(`- ${domain}: ${desc}`);
+  }
+
+  let section = lines.join("\n");
+  const charBudget = budgetTokens * CHARS_PER_TOKEN;
+  if (section.length > charBudget) {
+    section = section.substring(0, charBudget) + "\n...(truncated)";
+  }
+
+  return { boundariesSection: section, forbiddenDomains };
+}
+
+// ── Conversation history ───────────────────────────────────────────
+
+/**
+ * Load conversation history from interaction logs, filtered by disclosure boundaries.
+ */
+function loadConversationHistory(senderSlug, channelId, budgetTokens, forbiddenDomains) {
+  const candidateDirs = [];
+  if (channelId) candidateDirs.push(join(AGENT_REPO_DIR, "memory", "interactions", "slack", channelId));
+  if (senderSlug) {
+    candidateDirs.push(join(AGENT_REPO_DIR, "memory", "interactions", "slack", `dm-${senderSlug}`));
+    candidateDirs.push(join(AGENT_REPO_DIR, "memory", "interactions", "slack", senderSlug));
+  }
+
+  const entries = [];
+  const seenKeys = new Set();
+
+  for (const dir of candidateDirs) {
+    try {
+      const files = readdirSync(dir)
+        .filter((f) => f.endsWith(".jsonl"))
+        .sort()
+        .reverse()
+        .slice(0, 3); // Today + yesterday + day before
+
+      for (const file of files) {
+        try {
+          const content = readFileSync(join(dir, file), "utf-8").trim();
+          if (!content) continue;
+          const lines = content.split("\n").filter(Boolean);
+          for (const line of lines) {
+            try {
+              const entry = JSON.parse(line);
+              // Deduplicate by ts:from key
+              const dedupKey = `${entry.ts || ""}:${entry.from || entry.sender || ""}`;
+              if (seenKeys.has(dedupKey)) continue;
+              seenKeys.add(dedupKey);
+              entries.push(entry);
+            } catch { /* skip malformed lines */ }
+          }
+        } catch { /* skip unreadable files */ }
+      }
+    } catch { /* dir doesn't exist — try next */ }
+  }
+
+  if (entries.length === 0) return null;
+
+  // Sort chronologically
+  entries.sort((a, b) => {
+    const tsA = a.ts || a.received_at || a.timestamp || "";
+    const tsB = b.ts || b.received_at || b.timestamp || "";
+    return tsA.localeCompare(tsB);
+  });
+
+  // Filter by disclosure boundaries
+  const domainKeywords = buildDomainKeywords();
+  const filtered = entries.filter((entry) => {
+    const text = entry.content || entry.text || "";
+    return !matchesForbiddenDomain(text, forbiddenDomains, domainKeywords);
+  });
+
+  if (filtered.length === 0) return null;
+
+  // Truncate to character budget — most recent messages prioritised (iterate from end)
+  const charBudget = budgetTokens * CHARS_PER_TOKEN;
+  const outputLines = [];
+  let totalChars = 0;
+
+  for (let i = filtered.length - 1; i >= 0; i--) {
+    const entry = filtered[i];
+    const from = entry.from || entry.sender || entry.user_name || "unknown";
+    const text = entry.content || entry.text || "(no text)";
+    const ts = entry.received_at || entry.timestamp || entry.processed_at || entry.ts || "";
+    const responded = entry.response_sent ? " [Responded]" : "";
+    const line = `[${ts}] ${from}: ${text.substring(0, 500)}${responded}`;
+
+    if (totalChars + line.length > charBudget) break;
+    outputLines.unshift(line);
+    totalChars += line.length;
+  }
+
+  return outputLines.length > 0 ? outputLines.join("\n") : null;
+}
+
+// ── Active sessions ────────────────────────────────────────────────
+
+/**
+ * Load active sessions and format as bullet list.
+ */
+function loadActiveSessions(budgetTokens) {
+  try {
+    const filePath = join(SESSION_DIR, "active.json");
+    const raw = readFileSync(filePath, "utf-8");
+    const data = JSON.parse(raw);
+
+    // data may be an object with session entries or an array
+    const sessions = Array.isArray(data) ? data : Object.values(data);
+    if (sessions.length === 0) return null;
+
+    const lines = [];
+    const charBudget = budgetTokens * CHARS_PER_TOKEN;
+    let totalChars = 0;
+
+    for (const sess of sessions) {
+      const id = sess.sessionId || sess.id || "unknown";
+      const summary = sess.summary || sess.task || "no summary";
+      const model = sess.model || "unknown";
+      const sender = sess.sender || sess.owner || "unknown";
+      const started = sess.startTime || sess.started_at || "";
+      let age = "";
+      if (started) {
+        const ms = Date.now() - new Date(started).getTime();
+        const mins = Math.round(ms / 60000);
+        age = `${mins}m ago`;
+      }
+      const line = `- ${id}: ${summary} (model=${model}, sender=${sender}${age ? ", " + age : ""})`;
+      if (totalChars + line.length > charBudget) break;
+      lines.push(line);
+      totalChars += line.length;
+    }
+
+    return lines.length > 0 ? lines.join("\n") : null;
+  } catch {
+    return null;
+  }
+}
+
+// ── Sent registry ──────────────────────────────────────────────────
+
+/**
+ * Load recent sent-registry entries for the same channel.
+ */
+function loadSentRegistry(channelId, budgetTokens) {
+  if (!channelId) return null;
+  try {
+    const filePath = join(SESSION_DIR, "sent-registry.jsonl");
+    const raw = readFileSync(filePath, "utf-8").trim();
+    if (!raw) return null;
+
+    const allLines = raw.split("\n").filter(Boolean);
+    const matching = [];
+    for (const line of allLines) {
+      try {
+        const entry = JSON.parse(line);
+        if (entry.channelId === channelId || entry.channel_id === channelId || entry.channel === channelId) {
+          matching.push(entry);
+        }
+      } catch { /* skip malformed */ }
+    }
+
+    // Take last 20
+    const recent = matching.slice(-20);
+    if (recent.length === 0) return null;
+
+    const charBudget = budgetTokens * CHARS_PER_TOKEN;
+    const lines = [];
+    let totalChars = 0;
+
+    for (const entry of recent) {
+      const ts = entry.timestamp || entry.sent_at || "";
+      const type = entry.type || entry.action || "message";
+      const preview = (entry.preview || entry.content || entry.text || "").substring(0, 100);
+      const line = `[${ts}] (${type}) ${preview}`;
+      if (totalChars + line.length > charBudget) break;
+      lines.push(line);
+      totalChars += line.length;
+    }
+
+    return lines.length > 0 ? lines.join("\n") : null;
+  } catch {
+    return null;
+  }
+}
+
+// ── Org shared memory ──────────────────────────────────────────────
+
+// Cached org config (config/org.yaml) — read once per process. undefined until loaded.
+let cachedOrgCfg;
+
+/**
+ * Load the org config for the recall binding, cached for the process. Returns {}
+ * on any error so callers can null-check uniformly — the org binding must never
+ * take down the daemon's main path.
+ */
+function loadCachedOrgCfg() {
+  if (cachedOrgCfg !== undefined) return cachedOrgCfg;
+  try {
+    cachedOrgCfg = loadOrgConfig(AGENT_REPO_DIR) || {};
+  } catch {
+    cachedOrgCfg = {};
+  }
+  return cachedOrgCfg;
+}
+
+/** For tests: reset the cached org config so a fresh config/org.yaml is re-read. */
+export function _resetOrgCfgCache() {
+  cachedOrgCfg = undefined;
+}
+
+/**
+ * Derive a short recall query from the item / classification being processed.
+ * Prefers the classifier summary, then subject, then a slice of the body.
+ */
+function recallQueryFor(item, classResult, queueItem) {
+  const parts = [];
+  if (classResult && classResult.summary) parts.push(String(classResult.summary));
+  if (item && item.subject) parts.push(String(item.subject));
+  if (queueItem && queueItem.title) parts.push(String(queueItem.title));
+  if (parts.length === 0 && item && item.content) parts.push(String(item.content).slice(0, 200));
+  return parts.join(" — ").trim();
+}
+
+/**
+ * Read the org's shared memory (central store, `knowledge.search` RPC) for this
+ * item's topic and render it as a bounded, clearly-delimited section, or null
+ * when disabled / nothing relevant came back. Fail-open: never throws. The
+ * SERVER enforces ACL — we render exactly what recall() returns.
+ *
+ * @param {object} item
+ * @param {object} classResult
+ * @param {number} budgetTokens
+ * @param {object} [opts] - { cfg?, recallImpl?, queueItem? } — injectable for tests
+ * @returns {Promise<string|null>}
+ */
+async function loadOrgMemory(item, classResult, budgetTokens, opts = {}) {
+  try {
+    const cfg = opts.cfg !== undefined ? opts.cfg : loadCachedOrgCfg();
+    if (!orgEnabled(cfg)) return null;
+    const query = recallQueryFor(item, classResult, opts.queueItem);
+    if (!query) return null;
+
+    const recallImpl = opts.recallImpl || orgRecall;
+    const facts = await recallImpl(query, { cfg, limit: ORG_RECALL_MAX_FACTS });
+    if (!Array.isArray(facts) || facts.length === 0) return null;
+
+    const charBudget = budgetTokens * CHARS_PER_TOKEN;
+    const lines = [
+      "Facts the org already knows that may bear on this item (ACL-filtered by the",
+      "server; treat as trusted internal context):",
+    ];
+    let n = 0;
+    let totalChars = lines.join("\n").length;
+    for (const f of facts) {
+      if (n >= ORG_RECALL_MAX_FACTS) break;
+      const textRaw = typeof f === "string" ? f : (f && (f.text || f.content || f.body) || "");
+      const text = String(textRaw).trim();
+      if (!text) continue;
+      const clipped = text.length > ORG_RECALL_FACT_CHARS ? `${text.slice(0, ORG_RECALL_FACT_CHARS)}…` : text;
+      const id = f && typeof f === "object" && f.id != null ? ` [${String(f.id)}]` : "";
+      const line = `- ${clipped}${id}`;
+      if (totalChars + line.length > charBudget) break;
+      lines.push(line);
+      totalChars += line.length;
+      n += 1;
+    }
+    return n > 0 ? lines.join("\n") : null;
+  } catch {
+    // Fail-open: org recall must never take down the context build.
+    return null;
+  }
+}
+
+// ── Main export ────────────────────────────────────────────────────
+
+/**
+ * Compile all relevant context for a daemon session.
+ *
+ * @param {object} item - The inbox item or queue item being processed
+ * @param {object} classResult - Classification result with at least .priority
+ * @param {object} [options] - Optional overrides
+ * @returns {{ contextBlock: string, tokenBudget: number, tokenEstimate: number }}
+ */
+export async function compileContext(item, classResult, options = {}) {
+  try {
+    // Step 1 — Budget
+    const priority = classResult?.priority || "normal";
+    const tokenBudget = TOKEN_BUDGETS[priority] || TOKEN_BUDGETS.normal;
+
+    const senderSlug = item?.sender
+      ? item.sender.replace(/\s+/g, "-").toLowerCase()
+      : null;
+    const channelId = extractChannelId(item);
+
+    // Step 2 — User profile
+    const profileText = loadUserProfile(senderSlug, ALLOC.profile);
+
+    // Step 3 — Disclosure boundaries
+    const { boundariesSection, forbiddenDomains } = loadDisclosureBoundaries(senderSlug, ALLOC.boundaries);
+
+    // Step 4 — Conversation history
+    const fixedTokens = ALLOC.profile + ALLOC.boundaries + ALLOC.active_sessions + ALLOC.sent_registry + ALLOC.org_memory;
+    const remainingTokens = tokenBudget - fixedTokens;
+    const historyBudget = Math.floor(remainingTokens * HISTORY_SHARE);
+    const historyText = loadConversationHistory(senderSlug, channelId, historyBudget, forbiddenDomains);
+
+    // Step 5 — Active sessions
+    const sessionsText = loadActiveSessions(ALLOC.active_sessions);
+
+    // Step 6 — Sent registry
+    const sentText = loadSentRegistry(channelId, ALLOC.sent_registry);
+
+    // Step 6b — Org shared memory (central store via knowledge.* RPC). Routed
+    // alongside the local reads; bounded + fail-open (a disabled/unreachable
+    // server or empty recall injects nothing). The `cfg`/`recallImpl` overrides
+    // let tests drive this deterministically without a live server.
+    const orgMemoryText = await loadOrgMemory(item, classResult, ALLOC.org_memory, {
+      cfg: options.orgCfg,
+      recallImpl: options.orgRecall,
+      queueItem: options.queueItem,
+    });
+
+    // Step 7 — Assemble
+    const sections = [];
+    sections.push("--- COMPILED CONTEXT (do not repeat verbatim) ---");
+    sections.push("");
+
+    if (profileText) {
+      sections.push("## Sender Profile");
+      sections.push(profileText);
+      sections.push("");
+    }
+
+    if (boundariesSection) {
+      sections.push("## Disclosure Boundaries");
+      sections.push(boundariesSection);
+      sections.push("");
+    }
+
+    if (orgMemoryText) {
+      sections.push("## Org Shared Memory");
+      sections.push(orgMemoryText);
+      sections.push("");
+    }
+
+    sections.push("## Conversation History (most recent last)");
+    sections.push(historyText || "No prior conversation history found.");
+    sections.push("");
+
+    if (sessionsText) {
+      sections.push("## Active Sessions");
+      sections.push(sessionsText);
+      sections.push("");
+    }
+
+    if (sentText) {
+      sections.push("## Already Sent");
+      sections.push(sentText);
+      sections.push("");
+    }
+
+    sections.push("--- END CONTEXT ---");
+
+    const contextBlock = sections.join("\n");
+    const tokenEstimate = Math.ceil(contextBlock.length / CHARS_PER_TOKEN);
+
+    return { contextBlock, tokenBudget, tokenEstimate };
+  } catch (err) {
+    // Never throw — return minimal context on error
+    const tokenBudget = TOKEN_BUDGETS[classResult?.priority] || TOKEN_BUDGETS.normal;
+    const contextBlock = [
+      "--- COMPILED CONTEXT (do not repeat verbatim) ---",
+      "",
+      "## Conversation History (most recent last)",
+      "No prior conversation history found.",
+      "",
+      "--- END CONTEXT ---",
+    ].join("\n");
+    return {
+      contextBlock,
+      tokenBudget,
+      tokenEstimate: Math.ceil(contextBlock.length / CHARS_PER_TOKEN),
+    };
+  }
+}