@johpaz/hive 1.1.0

package/packages/core/src/agent/context-compiler.ts

@@ -0,0 +1,378 @@
+/**
+ * Context Compiler
+ *
+ * Compiled BEFORE every model call. Implements the 4 strategies:
+ *   WRITE    → scratchpad (persistent notes survive compression)
+ *   SELECT   → history (recent N), playbook rules (FTS5), tool loadout (3 levels)
+ *   COMPRESS → summaries (if conversation is long)
+ *   ISOLATE  → workers receive only their task context, not full history
+ *
+ * Constitutional ethics layer: always injected first, never filtered.
+ *
+ * Output structure (used directly by agent-loop.ts):
+ *   systemPrompt  — assembled from layers 1-4
+ *   messages      — history to send to model
+ *   tools         — LLMToolDef[] after 3-level filter
+ */
+
+import { getDb } from "../storage/sqlite"
+import { logger } from "../utils/logger"
+import type { LLMMessage, LLMToolDef } from "./llm-client"
+import type { Tool } from "./native-tools"
+import type { MCPClientManager } from "@johpaz/hive-mcp"
+import {
+  getRecentMessages,
+  getSummary,
+  getScratchpad,
+  getMessageCount,
+  toAPIMessages,
+  type StoredMessage,
+} from "./conversation-store"
+import {
+  collectNativeTools,
+  collectMCPTools,
+  filterToolsByAgent,
+  selectToolLoadout,
+  syncNativeToolsToDB,
+  toToolDefs,
+  getToolGroup,
+} from "./native-tools"
+import { getUserDate, getUserTime } from "../utils/date"
+import { encode } from "toon-format-parser"
+
+const log = logger.child("context-compiler")
+
+// ─── Configuration ─────────────────────────────────────────────────────────
+
+const RECENT_MSGS_LIMIT = 6       // keep last N messages for context (small models need small context)
+const PLAYBOOK_RULES_LIMIT = 5    // max playbook rules injected per turn
+const MAX_TOOL_GROUPS = 3         // max semantic groups loaded per turn (each group = all tools in category)
+
+// ─── Types ─────────────────────────────────────────────────────────────────
+
+export interface CompileOptions {
+  agentId: string
+  threadId: string
+  userMessage: string
+  channel?: string
+  mcpManager?: MCPClientManager | null
+  /** If true, this is a worker receiving an isolated task — don't inject full history */
+  isolated?: boolean
+  /** For isolated workers: the task description to inject instead of full history */
+  taskContext?: string
+}
+
+export interface CompiledContext {
+  systemPrompt: string
+  messages: LLMMessage[]
+  tools: LLMToolDef[]
+  allTools: Tool[]       // raw tool objects for execution
+}
+
+// ─── Main compiler ──────────────────────────────────────────────────────────
+
+export async function compileContext(opts: CompileOptions): Promise<CompiledContext> {
+  const t0 = performance.now()
+  const db = getDb()
+
+  // Load agent row
+  const agent = db.query<any, [string]>("SELECT * FROM agents WHERE id = ?").get(opts.agentId)
+  if (!agent) throw new Error(`Agent not found: ${opts.agentId}`)
+
+  // Load user
+  const user = db.query<any, [string]>(
+    "SELECT * FROM users WHERE id = (SELECT user_id FROM agents WHERE id = ? LIMIT 1)"
+  ).get(opts.agentId) ?? db.query<any, []>("SELECT * FROM users LIMIT 1").get()
+
+  // ── Tool collection FIRST — needed to generate dynamic capabilities ────
+  // Level 1: full catalog (native + MCP) — must happen before system prompt assembly
+  const nativeTools = collectNativeTools()
+  const mcpTools = opts.mcpManager
+    ? await collectMCPTools(opts.mcpManager)
+    : []
+  const allTools = [...nativeTools, ...mcpTools]
+
+  // Sync catalog to DB so FTS5 is current (fast, ~1ms for 50 tools)
+  syncNativeToolsToDB(allTools)
+
+  // Level 2: filter by agent assignment (tools_json NULL = all allowed)
+  const agentFiltered = filterToolsByAgent(allTools, agent.tools_json)
+
+  // Level 3: Group selection — always-include + top N semantic groups expanded
+  const loadout = selectToolLoadout(agentFiltered, opts.userMessage, MAX_TOOL_GROUPS)
+
+  const systemParts: string[] = []
+
+  // ── Layer 1: Ethics (constitutional, always first) ──────────────────────
+  const ethicsText = loadEthics()
+  if (ethicsText) {
+    systemParts.push(ethicsText)
+  }
+
+  // ── Layer 2: Agent identity ─────────────────────────────────────────────
+  let agentPrompt: string
+  if (agent.system_prompt) {
+    agentPrompt = agent.description
+      ? `# ROLE\n${agent.description}\n\n${agent.system_prompt}`
+      : agent.system_prompt
+  } else if (agent.description) {
+    agentPrompt = `# ROLE\n${agent.description}\n\nYou are a helpful assistant. Use your available tools to complete the user's requests.`
+  } else {
+    agentPrompt = "You are a helpful assistant. Use your available tools to complete the user's requests."
+  }
+  systemParts.push(agentPrompt)
+
+  // ── Layer 2b: Hive Capabilities Manifest (dynamic — from actual loaded tools) ─
+  // Only for coordinators — workers get minimal context
+  if (!opts.isolated) {
+    const capabilities = loadHiveCapabilities(allTools)
+    if (capabilities) {
+      systemParts.push(capabilities)
+    }
+  }
+
+  // ── Layer 2c: User profile ──────────────────────────────────────────────
+  if (user) {
+    const userInfo: Record<string, string> = {}
+    if (user.name) userInfo.name = user.name
+    if (user.occupation) userInfo.occupation = user.occupation
+    if (user.language) userInfo.language = user.language
+    if (user.timezone) userInfo.timezone = user.timezone
+    if (user.notes) userInfo.notes = user.notes
+    if (Object.keys(userInfo).length > 0) {
+      systemParts.push(encode({ user: userInfo }))
+    }
+  }
+
+  // ── Layer 3: Playbook rules (ACE, FTS5) ─────────────────────────────────
+  const playbookRules = loadPlaybookRules(opts.userMessage)
+  if (playbookRules.length > 0) {
+    systemParts.push(encode({ playbook: playbookRules }))
+  }
+
+  // ── Layer 4: Scratchpad (persistent notes) ──────────────────────────────
+  if (!opts.isolated) {
+    const notes = getScratchpad(opts.threadId)
+    if (notes.length > 0) {
+      const notesMap: Record<string, string> = {}
+      for (const n of notes) notesMap[n.key] = n.value
+      systemParts.push(encode({ notes: notesMap }))
+    }
+  }
+
+  // ── Layer 5: Isolated task context (for workers) ─────────────────────────
+  if (opts.isolated && opts.taskContext) {
+    systemParts.push(`# YOUR TASK\n${opts.taskContext}`)
+  }
+
+  // ── Layer 6: Environment / temporal info ────────────────────────────────
+  const now = new Date()
+  const tz = user?.timezone || "UTC"
+  let localTime = "N/A"
+  try {
+    localTime = now.toLocaleString("en-US", { timeZone: tz, dateStyle: "full", timeStyle: "long" })
+  } catch { /* bad TZ */ }
+
+  systemParts.push(encode({
+    env: {
+      agent_id: agent.id,
+      thread_id: opts.threadId,
+      utc: now.toISOString(),
+      local: `${localTime} (${tz})`,
+      date: getUserDate(tz, now),
+      time: getUserTime(tz, now),
+    }
+  }))
+
+  // ── Tool loadout summary (TOON, grouped) — current turn selection ────────
+  if (loadout.length > 0) {
+    const grouped: Record<string, string[]> = {}
+    for (const t of loadout) {
+      const cat = t.name.includes("__")
+        ? `mcp:${t.name.split("__")[0]}`
+        : getToolGroup(t.name)
+      if (!grouped[cat]) grouped[cat] = []
+      grouped[cat].push(t.name)
+    }
+    systemParts.push(encode({ tools: grouped }))
+  }
+
+  // ── History selection ───────────────────────────────────────────────────
+  let messages: LLMMessage[]
+
+  if (opts.isolated) {
+    // Workers only get the task description — no full conversation history
+    messages = []
+  } else {
+    const totalMessages = getMessageCount(opts.threadId)
+    const summary = getSummary(opts.threadId)
+
+    if (summary && totalMessages > RECENT_MSGS_LIMIT) {
+      // Inject summary as a system message, then recent messages
+      const recent = getRecentMessages(opts.threadId, RECENT_MSGS_LIMIT)
+      messages = [
+        { role: "system", content: `[Previous conversation summary]\n${summary.summary}` },
+        ...toAPIMessages(recent),
+      ]
+    } else {
+      // Short conversation — pass all messages
+      const recent = getRecentMessages(opts.threadId, RECENT_MSGS_LIMIT)
+      messages = toAPIMessages(recent)
+    }
+  }
+
+  log.info(
+    `[context-compiler] agent=${agent.name} tools=${loadout.length}/${allTools.length} ` +
+    `msgs=${messages.length} playbook=${playbookRules.length} ` +
+    `elapsed=${(performance.now() - t0).toFixed(1)}ms`
+  )
+
+  return {
+    systemPrompt: systemParts.join("\n\n"),
+    messages,
+    tools: toToolDefs(loadout),
+    allTools: loadout,
+  }
+}
+
+// ─── Ethics loader ──────────────────────────────────────────────────────────
+
+function loadEthics(): string {
+  try {
+    const db = getDb()
+    const rows = db.query<any, []>(
+      "SELECT content FROM ethics WHERE active = 1 ORDER BY id ASC"
+    ).all()
+    if (rows.length === 0) return ""
+    return encode({ ethics: rows.map((r: any) => r.content) })
+  } catch {
+    return ""
+  }
+}
+
+// ─── Playbook loader (FTS5) ─────────────────────────────────────────────────
+
+function loadPlaybookRules(userMessage: string): string[] {
+  try {
+    const db = getDb()
+
+    // Try FTS5 first
+    const ftsRows = db.query<any, [string, number]>(`
+      SELECT p.rule FROM playbook p
+      JOIN playbook_fts f ON p.id = f.rowid
+      WHERE playbook_fts MATCH ?
+        AND p.active = 1
+        AND p.helpful_count >= p.harmful_count
+      ORDER BY rank
+      LIMIT ?
+    `).all(sanitizeFTSQuery(userMessage), PLAYBOOK_RULES_LIMIT)
+
+    if (ftsRows.length > 0) {
+      return ftsRows.map((r: any) => r.rule)
+    }
+
+    // Fallback: most-used active rules
+    const fallback = db.query<any, [number]>(`
+      SELECT rule FROM playbook
+      WHERE active = 1 AND helpful_count >= harmful_count
+      ORDER BY helpful_count DESC
+      LIMIT ?
+    `).all(PLAYBOOK_RULES_LIMIT)
+
+    return fallback.map((r: any) => r.rule)
+  } catch {
+    return []
+  }
+}
+
+function sanitizeFTSQuery(text: string): string {
+  // FTS5 query: take first N unique words, escape special chars
+  const words = text
+    .replace(/['"*]/g, " ")
+    .split(/\s+/)
+    .filter((w) => w.length > 2)
+    .slice(0, 8)
+  return words.join(" OR ")
+}
+
+// ─── Hive Capabilities loader ───────────────────────────────────────────────
+
+/**
+ * Generates the Hive Capabilities manifest dynamically from:
+ *   - Static sections in `hive_capabilities` DB (architecture, agents, canvas, etc.)
+ *   - Native tools grouped by category (from allTools — always current)
+ *   - MCP tools grouped by server (from allTools — always current)
+ *
+ * The `tools_native` and `mcp` DB sections are intentionally skipped;
+ * those are replaced by the dynamic blocks built from allTools.
+ */
+function loadHiveCapabilities(allTools: Tool[]): string {
+  try {
+    const db = getDb()
+
+    // Load static architecture/context sections — skip tools & mcp (generated dynamically)
+    const rows = db.query<any, []>(
+      "SELECT section, title, content FROM hive_capabilities WHERE active = 1 ORDER BY sort_order ASC"
+    ).all()
+
+    const DYNAMIC_SECTIONS = new Set(["tools", "mcp"])
+    const sections: string[] = rows
+      .filter((r: any) => !DYNAMIC_SECTIONS.has(r.section))
+      .map((r: any) => `## ${r.title}\n${r.content}`)
+
+    // ── Dynamic: native tools grouped by category ────────────────────────
+    const nativeByGroup: Record<string, { name: string; description: string }[]> = {}
+    const mcpByServer: Record<string, string[]> = {}
+
+    for (const t of allTools) {
+      if (t.name.includes("__")) {
+        // MCP tool: "serverName__toolName"
+        const server = t.name.split("__")[0]
+        if (!mcpByServer[server]) mcpByServer[server] = []
+        mcpByServer[server].push(t.name.split("__")[1])
+      } else {
+        const cat = getToolGroup(t.name)
+        if (!nativeByGroup[cat]) nativeByGroup[cat] = []
+        nativeByGroup[cat].push({ name: t.name, description: t.description })
+      }
+    }
+
+    if (Object.keys(nativeByGroup).length > 0) {
+      const lines = ["## Herramientas Nativas Disponibles"]
+      for (const [cat, tools] of Object.entries(nativeByGroup)) {
+        lines.push(`\n**${cat.toUpperCase()}:**`)
+        for (const t of tools) {
+          lines.push(`- ${t.name}: ${t.description.split(".")[0].substring(0, 80)}`)
+        }
+      }
+      sections.push(lines.join("\n"))
+    }
+
+    // ── Dynamic: MCP servers — from DB (always) + loaded tools (if connected) ──
+    // Query DB so the agent always knows which servers exist, even if disconnected
+    try {
+      const dbServers = db.query<any, []>(
+        "SELECT name, transport, url, status, tools_count FROM mcp_servers WHERE enabled = 1"
+      ).all()
+
+      if (dbServers.length > 0) {
+        const lines = ["## Servidores MCP Configurados"]
+        for (const srv of dbServers) {
+          const loadedTools = mcpByServer[srv.name]
+          if (loadedTools && loadedTools.length > 0) {
+            lines.push(`\n**${srv.name}** [${srv.transport}] ✓ conectado — tools: ${loadedTools.join(", ")}`)
+          } else {
+            const endpoint = srv.url ? ` (${srv.url})` : ""
+            lines.push(`\n**${srv.name}** [${srv.transport}]${endpoint} — ${srv.status} (${srv.tools_count ?? 0} tools registradas)`)
+          }
+        }
+        sections.push(lines.join("\n"))
+      }
+    } catch { /* no mcp_servers table yet */ }
+
+    if (sections.length === 0) return ""
+    return `# HIVE — CAPACIDADES DEL SISTEMA\n${sections.join("\n\n")}`
+  } catch {
+    return ""
+  }
+}

package/packages/core/src/agent/context-guard.ts

@@ -0,0 +1,91 @@
+import type { Config } from "../config/loader.ts";
+import { logger } from "../utils/logger.ts";
+import type { Message } from "../agent/context.ts";
+
+export interface ContextGuardResult {
+  canProceed: boolean;
+  currentTokens: number;
+  maxTokens: number;
+  utilizationPercent: number;
+  needsCompaction: boolean;
+}
+
+export class ContextGuard {
+  private config: Config;
+  private log = logger.child("context-guard");
+
+  constructor(config: Config) {
+    this.config = config;
+  }
+
+  estimateTokens(messages: Message[]): number {
+    let total = 0;
+    
+    for (const msg of messages) {
+      total += Math.ceil(msg.content.length / 4);
+      
+      if (msg.name) {
+        total += Math.ceil(msg.name.length / 4);
+      }
+      
+      if (msg.toolCalls) {
+        for (const tc of msg.toolCalls) {
+          total += Math.ceil(tc.name.length / 4);
+          total += Math.ceil(JSON.stringify(tc.arguments).length / 4);
+        }
+      }
+    }
+    
+    return total;
+  }
+
+  check(messages: Message[], systemPrompt?: string): ContextGuardResult {
+    const maxTokens = this.config.agent?.context?.maxTokens || 128000;
+    const threshold = this.config.agent?.context?.compactionThreshold || 0.8;
+    
+    let currentTokens = this.estimateTokens(messages);
+    
+    if (systemPrompt) {
+      currentTokens += Math.ceil(systemPrompt.length / 4);
+    }
+
+    currentTokens += 500;
+    
+    const utilizationPercent = currentTokens / maxTokens;
+    const needsCompaction = utilizationPercent >= threshold;
+    const canProceed = currentTokens < maxTokens * 0.95;
+
+    this.log.debug(`Context check: ${currentTokens}/${maxTokens} tokens (${(utilizationPercent * 100).toFixed(1)}%)`);
+
+    return {
+      canProceed,
+      currentTokens,
+      maxTokens,
+      utilizationPercent,
+      needsCompaction,
+    };
+  }
+
+  shouldCompact(messages: Message[], systemPrompt?: string): boolean {
+    const result = this.check(messages, systemPrompt);
+    return result.needsCompaction;
+  }
+
+  getRecommendedAction(messages: Message[], systemPrompt?: string): "proceed" | "compact" | "error" {
+    const result = this.check(messages, systemPrompt);
+    
+    if (result.utilizationPercent >= 0.95) {
+      return "error";
+    }
+    
+    if (result.needsCompaction) {
+      return "compact";
+    }
+    
+    return "proceed";
+  }
+}
+
+export function createContextGuard(config: Config): ContextGuard {
+  return new ContextGuard(config);
+}

package/packages/core/src/agent/context.ts

@@ -0,0 +1,138 @@
+export interface SoulConfig {
+  identity: string;
+  personality: string;
+  boundaries: string;
+  instructions: string;
+  capabilities: string[];
+  raw: string;
+}
+
+export interface UserConfig {
+  id: string;
+  name: string;
+  language: string;
+  timezone: string;
+  activeProjects: string[];
+  preferences: Record<string, unknown> | string;
+  raw: string;
+}
+
+export interface EthicsConfig {
+  raw: string;
+  loadedAt: Date;
+  path: string;
+}
+
+export interface Message {
+  role: "system" | "user" | "assistant" | "tool";
+  content: string;
+  name?: string;
+  toolCallId?: string;
+  toolCalls?: ToolCall[];
+}
+
+export interface ToolCall {
+  id: string;
+  name: string;
+  arguments: Record<string, unknown>;
+}
+
+export interface ToolInfo {
+  id: string;
+  name: string;
+  description: string | null;
+  category: string | null;
+}
+
+export interface SkillInfo {
+  id: string;
+  name: string;
+  description: string | null;
+  source: string;
+}
+
+export interface McpServerInfo {
+  id: string;
+  name: string;
+  transport: string;
+  enabled: boolean;
+  active: boolean;
+}
+
+export interface ChannelInfo {
+  id: string;
+  type: string;
+  accountId: string;
+  status: string;
+}
+
+export interface CodeBridgeInfo {
+  id: string;
+  name: string;
+  cliCommand: string;
+  port: number;
+  active: boolean;
+}
+
+export interface ProviderInfo {
+  id: string;
+  name: string;
+  baseUrl: string | null;
+  hasApiKey: boolean;
+}
+
+export interface ModelInfo {
+  id: string;
+  name: string;
+  providerId: string;
+  contextWindow: number | null;
+}
+
+export interface ContextOptions {
+  ethics: EthicsConfig | null;
+  soul: SoulConfig;
+  user: UserConfig | null;
+  skills: string[];
+  memory: string[];
+  channel?: string;
+  maxTokens: number;
+  // New structured data from DB
+  tools?: ToolInfo[];
+  mcpServers?: McpServerInfo[];
+  channels?: ChannelInfo[];
+  codeBridge?: CodeBridgeInfo[];
+  providers?: ProviderInfo[];
+  models?: ModelInfo[];
+}
+
+
+
+export function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+export function truncateToTokenLimit(
+  messages: Message[],
+  maxTokens: number,
+  keepLastN = 4
+): { messages: Message[]; truncated: number } {
+  let totalTokens = 0;
+  let truncated = 0;
+
+  const result: Message[] = [];
+  const toKeep = messages.slice(-keepLastN);
+
+  for (const msg of messages.slice(0, -keepLastN)) {
+    const tokens = estimateTokens(msg.content);
+    if (totalTokens + tokens < maxTokens * 0.7) {
+      result.push(msg);
+      totalTokens += tokens;
+    } else {
+      truncated++;
+    }
+  }
+
+  result.push(...toKeep);
+
+  return { messages: result, truncated };
+}