@johpaz/hive 1.1.0

package/packages/core/src/agent/native-tools.ts

@@ -0,0 +1,505 @@
+/**
+ * Native tool registry — no LangChain dependencies.
+ *
+ * Collects all native tools and MCP tools as plain Tool objects.
+ * The agent loop calls execute() directly.
+ */
+
+import { logger } from "../utils/logger"
+import { loadConfig } from "../config/loader"
+import type { LLMToolDef } from "./llm-client"
+import type { MCPClientManager } from "@johpaz/hive-mcp"
+import { encode } from "toon-format-parser"
+
+const log = logger.child("native-tools")
+
+// ─── Tool types ───────────────────────────────────────────────────────────────
+
+export interface Tool {
+  name: string
+  description: string
+  parameters: Record<string, unknown>
+  execute: (params: Record<string, unknown>, config?: any) => Promise<unknown>
+}
+
+export interface ToolResult {
+  success: boolean
+  result?: unknown
+  error?: string
+}
+
+// ─── Tool name sanitization (Gemini requirement) ──────────────────────────────
+
+function sanitizeName(name: string): string {
+  let s = name.replace(/[^a-zA-Z0-9_.\-:]/g, "_")
+  if (!/^[a-zA-Z_]/.test(s)) s = "_" + s
+  return s.length > 64 ? s.substring(0, 64) : s
+}
+
+// ─── Collect all native tools ─────────────────────────────────────────────────
+
+export function collectNativeTools(): Tool[] {
+  const config = loadConfig()
+  const tools: Tool[] = []
+
+  // Lazy imports to avoid issues at module load time
+  const add = (t: Tool | null | undefined) => { if (t) tools.push(t) }
+
+  const { readTool, writeTool, editTool } = require("../tools/read")
+  add(readTool); add(writeTool); add(editTool)
+
+  const { createExecTool } = require("../tools/exec")
+  add(createExecTool(config))
+
+  const { createNotifyTool, createReportProgressTool } = require("../tools/notify")
+  add(createNotifyTool()); add(createReportProgressTool())
+
+  const { createCronTools } = require("../tools/cron")
+  for (const t of createCronTools(config)) add(t)
+
+  const { createCanvasTools } = require("../canvas/canvas-tools")
+  for (const t of createCanvasTools(config)) add(t)
+
+  const { createWebSearchTool, createWebFetchTool } = require("../tools/web")
+  add(createWebSearchTool(config)); add(createWebFetchTool(config))
+
+  const { createBrowserTools } = require("../tools/browser")
+  for (const t of createBrowserTools({})) add(t)
+
+  const { createProjectTools } = require("../tools/project")
+  for (const t of createProjectTools({})) add(t)
+
+  const { MemoryStore } = require("../memory/notes")
+  const {
+    memoryWriteTool, memoryReadTool, memoryListTool,
+    memorySearchTool, memoryDeleteTool
+  } = require("../tools/memory")
+  const memStore = new MemoryStore(config)
+  add(memoryWriteTool(memStore)); add(memoryReadTool(memStore))
+  add(memoryListTool(memStore)); add(memorySearchTool(memStore))
+  add(memoryDeleteTool(memStore))
+
+  const { createCodeBridgeTools } = require("../tools/codebridge")
+  for (const t of createCodeBridgeTools()) add(t)
+
+  const {
+    projectCreateTool, projectStartTool, projectUpdateTool,
+    projectDoneTool, projectFailTool, taskCreateTool, taskUpdateTool,
+  } = require("../tools/project-management")
+  add(projectCreateTool); add(projectStartTool); add(projectUpdateTool)
+  add(projectDoneTool); add(projectFailTool); add(taskCreateTool); add(taskUpdateTool)
+
+  // Native coordinator tools (no LangChain)
+  const { createAgentNative, saveNoteNative, findAgentNative, archiveAgentNative } = require("../tools/coordinator-tools")
+  add(createAgentNative); add(saveNoteNative); add(findAgentNative); add(archiveAgentNative)
+
+  log.info(`[native-tools] Collected ${tools.length} native tools`)
+  return tools
+}
+
+// ─── Collect MCP tools ────────────────────────────────────────────────────────
+
+export async function collectMCPTools(mcpManager: MCPClientManager): Promise<Tool[]> {
+  const { getDb } = await import("../storage/sqlite")
+  const db = getDb()
+  const servers = db.query<any, []>("SELECT * FROM mcp_servers WHERE enabled = 1").all()
+  const tools: Tool[] = []
+
+  for (const server of servers) {
+    try {
+      const serverTools = mcpManager.getServerTools(server.name)
+      if (!serverTools || serverTools.length === 0) {
+        log.debug(`[native-tools] MCP server '${server.name}' is configured but has no loaded tools (status: ${server.status})`)
+        continue
+      }
+      for (const mcpTool of serverTools) {
+        const toolName = sanitizeName(`${server.name}__${mcpTool.name}`)
+        tools.push({
+          name: toolName,
+          description: mcpTool.description ?? `Tool ${mcpTool.name} from ${server.name}`,
+          parameters: mcpTool.inputSchema ?? { type: "object", properties: {} },
+          execute: async (params: Record<string, unknown>) => {
+            const result = await mcpManager.callTool(server.name, mcpTool.name, params)
+            if (typeof result === "string") return result
+            try {
+              return encode(result)
+            } catch {
+              return JSON.stringify(result)
+            }
+          },
+        })
+      }
+    } catch (err) {
+      log.warn(`[native-tools] Failed to load MCP tools from ${server.name}:`, err)
+    }
+  }
+
+  log.info(`[native-tools] Collected ${tools.length} MCP tools from ${servers.length} configured servers`)
+  return tools
+}
+
+// ─── Filter tools by agent assignment ────────────────────────────────────────
+
+/**
+ * Level 2 filter: only keep tools the agent is assigned to.
+ * If tools_json is null → all tools are allowed.
+ */
+export function filterToolsByAgent(
+  tools: Tool[],
+  agentToolsJson: string | null
+): Tool[] {
+  if (!agentToolsJson) return tools
+  try {
+    const allowed = new Set<string>(JSON.parse(agentToolsJson))
+    return tools.filter((t) => allowed.has(t.name))
+  } catch {
+    return tools
+  }
+}
+
+// ─── Tool Group definitions ───────────────────────────────────────────────────
+//
+// Each group maps to a DB category (from _guessCategory) + trigger keywords.
+// Keywords are appended to EVERY tool in the group during FTS5 sync, so any
+// query matching these words surfaces at least one tool from the group →
+// then the entire group is expanded into the loadout.
+//
+// MCP servers are treated as dynamic groups: "mcp:<serverName>"
+// Each server's tools are all included when any tool from that server matches.
+
+const GROUP_KEYWORDS: Record<string, string> = {
+  scheduling: "programar recordar recordatorio reminder agendar agenda schedule cron alarm alarma diario semanal mensual recurrente recurring tiempo futuro",
+  projects:   "proyecto tarea plan organizar milestone asignar equipo project task planning assign work tracking kanban sprint backlog",
+  filesystem: "archivo file leer escribir editar contenido document read write edit content source código fuente abrir guardar",
+  web:        "buscar internet google web search fetch download noticias news información investigar research http url scrape navegar",
+  browser:    "navegador browser click screenshot form formulario navegar scrape automatizar web página abrir abrir visitar UI",
+  memory:     "recordar nota guardar memoria anotar note save memory remember store retrieve knowledge persistir base conocimiento",
+  code:       "código code ejecutar run script bash terminal python javascript exec compile test deploy devops consola comando",
+  canvas:     "canvas diagrama graph visualizar nodo edge diagram visualization flow chart mapa mental mind map",
+  agents:     "agente worker especialista crear delegar hire assign sub-task coordinator manager contratar equipo",
+  core:       "notificar avisar progreso nota guardar comunicar",
+}
+
+// ─── Tool-specific synonym map (extra keywords per individual tool) ───────────
+const TOOL_SYNONYMS: Record<string, string> = {
+  // Scheduling
+  cron_add:    "nueva tarea programada nueva alarma set alarm",
+  cron_list:   "listar ver mostrar lista tareas programadas",
+  cron_remove: "eliminar borrar cancelar delete remove cancel",
+  cron_edit:   "modificar cambiar update patch",
+  // Projects
+  project_create: "nuevo proyecto nuevo plan create new",
+  project_update: "actualizar estado update status",
+  project_done:   "completar terminar cerrar complete finish done",
+  project_fail:   "falló error problema issue failed",
+  task_create:    "nueva tarea subtarea new subtask paso step",
+  task_update:    "avance progreso progress paso completado",
+  // Filesystem
+  read:  "ver mostrar show contenido content open",
+  write: "crear nuevo new save",
+  edit:  "modificar cambiar patch cambio diff",
+  exec:  "correr proceso bash shell comando",
+  // Web
+  web_search: "google busqueda encuentra find informacion noticias news",
+  web_fetch:  "descargar download pagina página html",
+  // Memory
+  save_note:     "apuntar anotar keep recordar",
+  memory_write:  "almacenar guardar store",
+  memory_read:   "recuperar retrieve obtener get",
+  memory_list:   "ver notas show list",
+  memory_search: "encontrar find buscar nota note",
+  memory_delete: "borrar eliminar remove",
+  // Browser
+  browser_open:       "abrir visitar navigate visit",
+  browser_screenshot: "captura foto imagen capture image",
+  browser_click:      "hacer clic presionar click press button",
+  browser_type:       "escribir teclear input",
+  browser_close:      "cerrar close",
+  // Agents
+  create_agent:     "nuevo agente especializado specialized nuevo worker",
+  report_progress:  "informar avance estado progress",
+  notify:           "enviar mensaje send message alerta alert",
+  // Canvas
+  canvas_update: "grafico diagrama nodo edge",
+  // Code bridge
+  bridge_exec:      "correr ejecutar local development terminal",
+  code_bridge_exec: "código run local development",
+}
+
+// ─── Sync native + MCP tools to DB catalog (for FTS5 search) ─────────────────
+
+/**
+ * Syncs ALL tools (native + MCP) into `tools` table + `tools_fts`.
+ * Descriptions are enriched with bilingual synonyms so FTS5 matches
+ * regardless of the user's language or phrasing.
+ * Called once per context compilation (~1ms for 80 tools).
+ */
+export function syncNativeToolsToDB(tools: Tool[]): void {
+  try {
+    const { getDb } = require("../storage/sqlite")
+    const db = getDb()
+
+    db.run("BEGIN")
+    try {
+      const upsert = db.prepare(`
+        INSERT INTO tools (id, name, description, category, enabled, active)
+        VALUES (?, ?, ?, ?, 1, 1)
+        ON CONFLICT(name) DO UPDATE SET
+          description = excluded.description,
+          category    = excluded.category,
+          enabled     = 1,
+          active      = 1
+      `)
+
+      db.run("DELETE FROM tools_fts")
+      const insertFts = db.prepare(
+        "INSERT INTO tools_fts(tool_name, name, description, category) VALUES (?, ?, ?, ?)"
+      )
+
+      for (const tool of tools) {
+        const cat = _guessCategory(tool.name)
+        // Enrich FTS5 description with synonyms (native tools use TOOL_SYNONYMS,
+        // MCP tools get category-based synonyms appended to their own description)
+        const enriched = _enrichDescription(tool.name, tool.description ?? "", cat)
+        upsert.run(tool.name, tool.name, tool.description ?? "", cat)
+        insertFts.run(tool.name, tool.name, enriched, cat)
+      }
+
+      db.run("COMMIT")
+    } catch (err) {
+      db.run("ROLLBACK")
+      throw err
+    }
+  } catch (err) {
+    log.warn("[native-tools] syncNativeToolsToDB failed (non-fatal):", err)
+  }
+}
+
+function _enrichDescription(name: string, description: string, category: string): string {
+  const parts: string[] = [description]
+
+  // 1. Tool-specific synonyms (extra precision per individual tool)
+  const toolSynonyms = TOOL_SYNONYMS[name]
+  if (toolSynonyms) parts.push(toolSynonyms)
+
+  // 2. Group keywords — appended to ALL tools in the category.
+  //    This ensures any tool in the group can trigger the full group expansion.
+  const groupKeywords = GROUP_KEYWORDS[category] || GROUP_KEYWORDS["core"]
+  if (groupKeywords) parts.push(groupKeywords)
+
+  // 3. MCP tools: derive keywords from server name + tool suffix
+  if (name.includes("__")) {
+    const [server, toolSuffix = ""] = name.split("__")
+    const MCP_SUFFIX_KEYWORDS: Record<string, string> = {
+      search:   "buscar search find encontrar web internet",
+      fetch:    "descargar fetch download url http",
+      email:    "email correo mail enviar send",
+      calendar: "calendario agenda schedule programar",
+      database: "base datos database query consulta sql",
+      file:     "archivo file documento read write",
+      code:     "código code ejecutar run script",
+      list:     "listar list show ver mostrar",
+      create:   "crear create nuevo new add",
+      delete:   "borrar delete remove eliminar",
+      update:   "actualizar update edit modificar",
+    }
+    for (const [key, extra] of Object.entries(MCP_SUFFIX_KEYWORDS)) {
+      if (toolSuffix.includes(key) || description.toLowerCase().includes(key)) {
+        parts.push(extra)
+        break
+      }
+    }
+    // Server name as keyword so "usa el servidor X" always matches
+    parts.push(server.replace(/_/g, " "))
+  }
+
+  return parts.join(" ")
+}
+
+export function getToolGroup(name: string): string {
+  return _guessCategory(name)
+}
+
+function _guessCategory(name: string): string {
+  if (name.startsWith("cron")) return "scheduling"
+  if (name.startsWith("project") || name.startsWith("task")) return "projects"
+  if (name.startsWith("browser")) return "browser"
+  if (name.startsWith("memory") || name === "save_note") return "memory"
+  if (["read", "write", "edit"].includes(name)) return "filesystem"
+  if (name === "web_search" || name === "web_fetch") return "web"
+  if (name.startsWith("canvas")) return "canvas"
+  if (name.startsWith("code") || name === "exec") return "code"
+  return "core"
+}
+
+// ─── Level 3: Tool Loadout via Group-based FTS5 ───────────────────────────────
+
+/**
+ * Tools always included regardless of message content (ALWAYS_INCLUDE).
+ * These never consume optional group slots.
+ * Max total loadout = ALWAYS_INCLUDE + expanded groups, capped at MAX_TOTAL_TOOLS.
+ */
+const CORE_TOOL_NAMES = new Set([
+  // Comms
+  "save_note", "notify", "report_progress",
+  // Filesystem
+  "read", "write", "edit",
+  // Cron/scheduling
+  "cron_add", "cron_list", "cron_remove", "cron_edit",
+  // Projects & tasks
+  "project_create", "task_create", "task_update",
+  "project_done", "project_fail", "project_update",
+  // Agent management
+  "create_agent", "find_agent",
+])
+
+/** Hard cap on total tools sent to the model (critical for low-resource local models) */
+const MAX_TOTAL_TOOLS = 20
+
+/**
+ * Level 3 selection — Group Loading.
+ *
+ * Instead of picking individual tools, FTS5 identifies which GROUPS are
+ * relevant, then expands to ALL tools in those groups. This gives the model
+ * a coherent, complete toolset per semantic domain.
+ *
+ * Native groups  → DB category (from _guessCategory)
+ * MCP groups     → one group per server ("mcp:<serverName>")
+ *
+ * Examples:
+ *   "hola"                     → ALWAYS_INCLUDE only (18 tools)
+ *   "busca noticias"           → ALWAYS_INCLUDE + web group (web_search + web_fetch)
+ *   "recuérdame el médico"     → ALWAYS_INCLUDE (cron tools already included)
+ *   "lee y edita config.yaml"  → ALWAYS_INCLUDE (read/write/edit already included)
+ *   "usa el servidor X"        → ALWAYS_INCLUDE + mcp:X group (all X tools)
+ */
+export function selectToolLoadout(
+  tools: Tool[],
+  userMessage: string,
+  maxGroups = 3
+): Tool[] {
+  const coreTools = tools.filter((t) => CORE_TOOL_NAMES.has(t.name))
+  const candidates = tools.filter((t) => !CORE_TOOL_NAMES.has(t.name))
+
+  if (candidates.length === 0) return coreTools
+
+  const ftsQuery = _buildFTSQuery(userMessage)
+  if (!ftsQuery) return coreTools   // purely conversational
+
+  // FTS5: fetch more results than needed to find enough distinct groups
+  let ftsRows: { tool_name: string; category: string }[] = []
+  try {
+    const { getDb } = require("../storage/sqlite")
+    const db = getDb()
+    ftsRows = db.query(`
+      SELECT tool_name, category FROM tools_fts
+      WHERE tools_fts MATCH ?
+      ORDER BY rank
+      LIMIT 40
+    `).all(ftsQuery) as { tool_name: string; category: string }[]
+  } catch {
+    return coreTools  // FTS5 not ready yet (pre-first-sync)
+  }
+
+  // Extract groups in FTS5 rank order (first occurrence = best score)
+  const rankedGroups: string[] = []
+  const seenGroups = new Set<string>()
+  for (const row of ftsRows) {
+    const group = row.tool_name.includes("__")
+      ? `mcp:${row.tool_name.split("__")[0]}`
+      : row.category
+    if (!seenGroups.has(group)) {
+      seenGroups.add(group)
+      rankedGroups.push(group)
+    }
+  }
+
+  // Keep only top N groups
+  const topGroups = new Set(rankedGroups.slice(0, maxGroups))
+
+  if (topGroups.size === 0) return coreTools
+
+  // Expand: include ALL tools belonging to each selected group
+  const expanded = candidates.filter((t) => {
+    if (t.name.includes("__")) {
+      return topGroups.has(`mcp:${t.name.split("__")[0]}`)
+    }
+    return topGroups.has(_guessCategory(t.name))
+  })
+
+  // Cap total tools — trim expanded if needed (core tools are never trimmed)
+  const remaining = MAX_TOTAL_TOOLS - coreTools.length
+  const trimmedExpanded = expanded.slice(0, Math.max(0, remaining))
+
+  log.debug(
+    `[native-tools] Groups: [${[...topGroups].join(", ")}] ` +
+    `expanded=${expanded.length} trimmed=${trimmedExpanded.length} core=${coreTools.length}`
+  )
+
+  return [...coreTools, ...trimmedExpanded]
+}
+
+function _buildFTSQuery(text: string): string {
+  const STOPWORDS = new Set([
+    "que", "con", "para", "por", "una", "uno", "los", "las", "del",
+    "como", "esta", "esto", "ese", "eso", "the", "and", "for",
+    "with", "this", "that", "have", "will", "also",
+  ])
+  const words = text
+    .toLowerCase()
+    .replace(/['"*()[\]]/g, " ")
+    .split(/\s+/)
+    .filter((w) => w.length > 2 && !STOPWORDS.has(w))
+    .slice(0, 8)
+  return words.join(" OR ")
+}
+
+// ─── Convert tools to LLM tool definitions ───────────────────────────────────
+
+export function toToolDefs(tools: Tool[]): LLMToolDef[] {
+  return tools.map((t) => ({
+    type: "function" as const,
+    function: {
+      name: sanitizeName(t.name),
+      description: t.description,
+      parameters: t.parameters as Record<string, unknown>,
+    },
+  }))
+}
+
+// ─── Execute a tool by name ───────────────────────────────────────────────────
+
+export async function executeTool(
+  tools: Tool[],
+  toolName: string,
+  argsJson: string,
+  context: { user_id: string; thread_id: string; channel?: string }
+): Promise<string> {
+  const tool = tools.find((t) => sanitizeName(t.name) === toolName || t.name === toolName)
+  if (!tool) {
+    const available = tools.map((t) => t.name).join(", ")
+    return `[Tool Error] Unknown tool: "${toolName}". Available tools: ${available}`
+  }
+
+  let args: Record<string, unknown>
+  try {
+    args = JSON.parse(argsJson || "{}")
+  } catch {
+    args = {}
+  }
+
+  try {
+    log.info(`[native-tools] Executing ${toolName}`, { args })
+    const result = await tool.execute(args, { configurable: context })
+    if (typeof result === "string") return result
+    try {
+      return encode(result)
+    } catch {
+      return JSON.stringify(result)
+    }
+  } catch (err) {
+    const msg = (err as Error).message
+    log.error(`[native-tools] Tool ${toolName} failed: ${msg}`)
+    return `[Tool Error] ${toolName}: ${msg}`
+  }
+}