@johpaz/hive-sdk 0.1.4 → 0.1.5

package/packages/core/src/mcp/tool-sync.ts

@@ -0,0 +1,138 @@
+/**
+ * MCP Tool Sync — Persist MCP tool definitions to DB and the HiveDB index
+ *
+ * When an MCP server connects, its tool definitions are persisted to
+ * the `mcpTools` collection and indexed in the HiveDB capability index
+ * for search_knowledge. When the server disconnects, all its tools are
+ * deleted from both (the index side uses a `server_id` filter, so only
+ * that server's documents are removed).
+ *
+ * This enables:
+ * 1. search_knowledge to find MCP tools via HiveDB BM25
+ * 2. Tool definitions to survive across context compiler invocations
+ * 3. Offline visibility of what MCP tools were available
+ */
+
+import { col } from "../storage/hive"
+import type { McpToolDoc } from "../storage/collections"
+import { logger } from "../utils/logger"
+import { mcpToolFullName } from "../agent/tool-selector"
+import {
+    replaceCapabilityDocs,
+    deleteCapabilitiesByServer,
+    type CapabilityDoc,
+} from "../agent/capability-search"
+
+const log = logger.child("mcp:tool-sync")
+
+export interface MCPToolDefinition {
+    name: string
+    description: string
+    inputSchema?: Record<string, unknown>
+}
+
+/**
+ * Generate a stable ID for an MCP tool based on server + tool name.
+ *
+ * MUST be identical to the LLM function name the agent calls — agent-loop
+ * resolves search results against executors by this exact string — so it
+ * delegates to mcpToolFullName instead of duplicating the algorithm.
+ */
+function mcpToolId(serverName: string, toolName: string): string {
+    return mcpToolFullName(serverName, toolName)
+}
+
+/**
+ * Persist MCP tool definitions to the `mcpTools` collection.
+ * Called when a server connects or reconnects.
+ * Replaces existing tools for the server first, then inserts fresh data.
+ */
+export async function syncMCPToolsToDB(
+    serverId: string,
+    serverName: string,
+    tools: MCPToolDefinition[]
+): Promise<void> {
+    try {
+        const mcpToolsCol = await col<McpToolDoc>("mcpTools")
+        const existing = await mcpToolsCol.findBy("server_id", serverId)
+        for (const e of existing) await mcpToolsCol.delete(e.id)
+
+        if (tools.length === 0) {
+            log.debug(`[mcp:tool-sync] No tools to persist for server ${serverName}`)
+            return
+        }
+
+        let count = 0
+        for (const tool of tools) {
+            const id = mcpToolId(serverName, tool.name)
+            await mcpToolsCol.put(id, {
+                id, server_id: serverId, server_name: serverName, tool_name: tool.name,
+                description: tool.description || "", category: "mcp", active: true,
+            })
+            count++
+        }
+
+        log.info(`[mcp:tool-sync] Persisted ${count} MCP tools for server ${serverName} to mcpTools`)
+    } catch (err) {
+        log.error(`[mcp:tool-sync] Failed to persist MCP tools for server ${serverName}:`, err)
+    }
+}
+
+/**
+ * Sync all active MCP tools from the `mcpTools` collection to the HiveDB
+ * capability index. Called after syncMCPToolsToDB or after startup to ensure
+ * the index is in sync.
+ *
+ * This does a full replace of `type=mcp` documents to avoid drift.
+ */
+export async function syncMCPToolsToIndex(): Promise<void> {
+    try {
+        const mcpToolsCol = await col<McpToolDoc>("mcpTools")
+        const mcpTools = (await mcpToolsCol.scan({})).map(e => e.doc).filter(t => t.active)
+
+        // Searchable text is the tool's OWN name/description only. The server
+        // name is deliberately NOT indexed: generic server names (e.g.
+        // "herramientas administrativas") would make every query matching the
+        // server name flood the results with that server's entire tool set.
+        // The server stays reachable via the server_id filter and appears in
+        // hydrated results.
+        //
+        // camelCase names ("crearEvento") tokenize as a single word, so a
+        // space-split variant goes into the searchable text ("crear Evento").
+        const splitCamel = (s: string) => s.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+        const docs: CapabilityDoc[] = mcpTools.map(tool => ({
+            type: "mcp" as const,
+            rawId: tool.id,
+            name: `${tool.tool_name} ${splitCamel(tool.tool_name)}`,
+            body: tool.description || tool.tool_name,
+            tags: tool.category ?? "",
+            extraFilters: [{ field: "server_id", value: tool.server_id }],
+        }))
+
+        await replaceCapabilityDocs("mcp", docs)
+
+        log.info(`[mcp:tool-sync] Synced ${mcpTools.length} MCP tools to HiveDB index`)
+    } catch (err) {
+        log.error(`[mcp:tool-sync] Failed to sync MCP tools to HiveDB index:`, err)
+    }
+}
+
+/**
+ * Delete all MCP tool definitions for a server from both the `mcpTools`
+ * collection and the HiveDB capability index. Called when a server
+ * disconnects or is removed.
+ */
+export async function clearMCPToolsFromDB(serverId: string): Promise<void> {
+    try {
+        const mcpToolsCol = await col<McpToolDoc>("mcpTools")
+        const existing = await mcpToolsCol.findBy("server_id", serverId)
+        for (const e of existing) await mcpToolsCol.delete(e.id)
+
+        // Remove only this server's documents from the index
+        await deleteCapabilitiesByServer(serverId)
+
+        log.info(`[mcp:tool-sync] Cleared MCP tools for server_id=${serverId}`)
+    } catch (err) {
+        log.error(`[mcp:tool-sync] Failed to clear MCP tools for server_id=${serverId}:`, err)
+    }
+}

package/packages/core/src/memory/Scratchpad.test.ts

@@ -1,33 +1,35 @@
-import { describe, it, expect, beforeAll, afterAll } from "bun:test";
+process.env.HIVE_DB_PATH = ":memory:";
+
+import { describe, it, expect, beforeEach, afterEach } from "bun:test";
 import { Scratchpad } from "./Scratchpad.ts";
-import { initializeDatabase, dbService } from "../storage/SQLiteStorage.ts";
-import { getHiveDB, closeHiveDB } from "../storage/HiveDBStorage.ts";
+import { closeHiveDb } from "../storage/hivedb.ts";
+import { ensureHiveDb } from "../storage/bootstrap.ts";
+import { getScratchpad } from "../agent/conversation-store.ts";
 
-describe("Scratchpad", () => {
-	let pad: Scratchpad;
+const THREAD = "test-thread";
 
-	beforeAll(async () => {
-		initializeDatabase();
-		await getHiveDB();
-		pad = new Scratchpad();
-	});
+let pad: Scratchpad;
 
-	afterAll(() => {
-		closeHiveDB();
-		dbService.close();
-	});
+beforeEach(async () => {
+	closeHiveDb();
+	await ensureHiveDb();
+	pad = new Scratchpad();
+});
 
-	const THREAD = "test-thread";
+afterEach(() => {
+	closeHiveDb();
+});
 
+describe("Scratchpad", () => {
 	it("writes and reads a note", async () => {
 		await pad.write(THREAD, "test-1", "hello world");
-		const value = await pad.read(THREAD, "test-1");
-		expect(value).toBe("hello world");
+		expect(await pad.read(THREAD, "test-1")).toBe("hello world");
 	});
 
 	it("lists notes as key-value map", async () => {
 		await pad.write(THREAD, "list-a", "aaa");
 		await pad.write(THREAD, "list-b", "bbb");
+
 		const notes = await pad.list(THREAD);
 		expect(notes["list-a"]).toBe("aaa");
 		expect(notes["list-b"]).toBe("bbb");
@@ -36,14 +38,31 @@ describe("Scratchpad", () => {
 	it("deletes a note", async () => {
 		await pad.write(THREAD, "to-delete", "delete me");
 		await pad.delete(THREAD, "to-delete");
-		const value = await pad.read(THREAD, "to-delete");
-		expect(value).toBeUndefined();
+		expect(await pad.read(THREAD, "to-delete")).toBeUndefined();
 	});
 
 	it("clear removes all notes for a thread", async () => {
 		await pad.write(THREAD, "clear-a", "a");
 		await pad.write(THREAD, "clear-b", "b");
 		await pad.clear(THREAD);
-		expect(Object.keys(await pad.list(THREAD)).length).toBe(0);
+		expect(Object.keys(await pad.list(THREAD))).toHaveLength(0);
+	});
+
+	it("no toca las notas de otros threads", async () => {
+		await pad.write("thread-uno", "k", "uno");
+		await pad.write("thread-dos", "k", "dos");
+		await pad.clear("thread-uno");
+
+		expect(await pad.read("thread-dos", "k")).toBe("dos");
+	});
+
+	it("escribe el mismo documento que lee el context compiler", async () => {
+		// La clase y `conversation-store` comparten colección e id. Antes la clase
+		// guardaba un doc incompleto (sin `source`, `createdAt` ni `seq`), así que
+		// una nota escrita por acá se ordenaba mal dentro del prompt.
+		await pad.write(THREAD, "compartida", "valor", "test");
+
+		const notes = await getScratchpad(THREAD);
+		expect(notes).toEqual([{ key: "compartida", value: "valor" }]);
 	});
 });

package/packages/core/src/memory/Scratchpad.ts

@@ -1,54 +1,47 @@
-import { getHiveDB } from "../storage/HiveDBStorage.ts";
+/**
+ * Scratchpad — notas persistentes por conversación.
+ *
+ * Es una fachada sobre las funciones de `agent/conversation-store.ts`, que son
+ * las que realmente usa el context compiler. Antes esta clase tenía su propia
+ * implementación contra la colección `scratchpad` y, como usaba el mismo id
+ * (`<threadId>:<key>`), escribía las mismas filas con un documento incompleto:
+ * sin `source`, sin `createdAt` y sin `seq`. Una nota guardada por acá quedaba
+ * sin el desempate por recencia que aplica `getScratchpad`, así que aparecía en
+ * un orden arbitrario dentro del prompt.
+ */
 
-interface ScratchpadDoc {
-  threadId: string;
-  key: string;
-  value: string;
-  updatedAt: number;
-}
+import {
+  saveScratchpadNote,
+  getScratchpad,
+  deleteScratchpadNote,
+} from "../agent/conversation-store.ts";
 
 export class Scratchpad {
-  async write(threadId: string, key: string, value: string): Promise<void> {
-    const db = await getHiveDB();
-    const col = db.collection<ScratchpadDoc>("scratchpad");
-    await col.put(this.docId(threadId, key), { threadId, key, value, updatedAt: Date.now() });
+  async write(threadId: string, key: string, value: string, source?: string): Promise<void> {
+    await saveScratchpadNote(threadId, key, value, source);
   }
 
   async read(threadId: string, key: string): Promise<string | undefined> {
-    const db = await getHiveDB();
-    const col = db.collection<ScratchpadDoc>("scratchpad");
-    const entry = await col.get(this.docId(threadId, key));
-    return entry?.doc.value;
+    const notes = await getScratchpad(threadId);
+    return notes.find((n) => n.key === key)?.value;
   }
 
+  /** Notas del thread, más recientes primero. */
   async list(threadId: string): Promise<Record<string, string>> {
-    const db = await getHiveDB();
-    const col = db.collection<ScratchpadDoc>("scratchpad");
-    const entries = await col.scan();
     const result: Record<string, string> = {};
-    for (const e of entries) {
-      if (e.doc.threadId === threadId) {
-        result[e.doc.key] = e.doc.value;
-      }
+    for (const note of await getScratchpad(threadId)) {
+      result[note.key] = note.value;
     }
     return result;
   }
 
   async delete(threadId: string, key: string): Promise<void> {
-    const db = await getHiveDB();
-    const col = db.collection<ScratchpadDoc>("scratchpad");
-    await col.delete(this.docId(threadId, key));
+    await deleteScratchpadNote(threadId, key);
   }
 
   async clear(threadId: string): Promise<void> {
-    const db = await getHiveDB();
-    const col = db.collection<ScratchpadDoc>("scratchpad");
-    const entries = await col.scan();
-    const ids = entries.filter(e => e.doc.threadId === threadId).map(e => e.id);
-    await db.batch(ids.map(id => ({ op: "delete" as const, collection: "scratchpad", id })));
-  }
-
-  private docId(threadId: string, key: string): string {
-    return `${threadId}:${key}`;
+    for (const note of await getScratchpad(threadId)) {
+      await deleteScratchpadNote(threadId, note.key);
+    }
   }
 }

package/packages/core/src/multimodal/vision-service.ts

@@ -1,4 +1,5 @@
-import { getDb } from "../storage/SQLiteStorage"
+import { col } from "../storage/hive"
+import type { ChannelDoc, ModelDoc, ProviderDoc } from "../storage/collections"
 import { loadProviderApiKey } from "../storage/crypto"
 import { logger } from "../utils/logger"
 import type { ImageInput, DocumentInput, VisionConfig } from "./types"
@@ -18,27 +19,19 @@ class MultimodalService {
     return MultimodalService.instance
   }
 
-  getChannelVisionConfig(channelId: string): VisionConfig {
-    const db = getDb()
-    const result = db.query(`
-      SELECT vision_enabled, ocr_provider, vision_provider, vision_model_id
-      FROM channels WHERE id = ?
-    `).get(channelId) as {
-      vision_enabled: number
-      ocr_provider: string | null
-      vision_provider: string | null
-      vision_model_id: string | null
-    } | undefined
-
-    if (!result) {
+  async getChannelVisionConfig(channelId: string): Promise<VisionConfig> {
+    const channelsCol = await col<ChannelDoc>("channels")
+    const entry = await channelsCol.get(channelId)
+
+    if (!entry) {
       return { visionEnabled: false, ocrProvider: null, visionProvider: null, visionModelId: null }
     }
 
     return {
-      visionEnabled: result.vision_enabled === 1,
-      ocrProvider: result.ocr_provider,
-      visionProvider: result.vision_provider,
-      visionModelId: result.vision_model_id,
+      visionEnabled: entry.doc.vision_enabled,
+      ocrProvider: entry.doc.ocr_provider,
+      visionProvider: entry.doc.vision_provider,
+      visionModelId: entry.doc.vision_model_id,
     }
   }
 
@@ -131,11 +124,26 @@ class MultimodalService {
   }
 
   private async getProviderApiKey(providerId: string): Promise<string | null> {
-    const db = getDb()
     const apiKey = await loadProviderApiKey(providerId)
     return apiKey || null
   }
 
+  /** Modelo con capacidad de visión del provider según la BD; el literal queda como último recurso. */
+  private async getVisionModel(providerId: string, fallback: string): Promise<string> {
+    try {
+      const modelsCol = await col<ModelDoc>("models")
+      const models = await modelsCol.findBy("provider_id", providerId)
+      const candidates = models.filter(e =>
+        e.doc.model_type === "llm" &&
+        (e.doc.capabilities?.includes('"vision"') || e.doc.capabilities?.includes('"ocr"'))
+      )
+      const best = [...candidates].sort((a, b) => Number(b.doc.active) - Number(a.doc.active))[0]
+      return best?.doc.id || fallback
+    } catch {
+      return fallback
+    }
+  }
+
   private async ocrWithOpenAI(image: ImageInput): Promise<string> {
     const key = await this.getProviderApiKey("openai") || process.env.OPENAI_API_KEY
     if (!key) throw new Error("OPENAI_API_KEY not configured for OCR")
@@ -146,7 +154,7 @@ class MultimodalService {
       method: "POST",
       headers: { "Content-Type": "application/json", "Authorization": `Bearer ${key}` },
       body: JSON.stringify({
-        model: "gpt-4o-mini",
+        model: await this.getVisionModel("openai", "gpt-4o-mini"),
         messages: [{
           role: "user",
           content: [
@@ -183,7 +191,7 @@ class MultimodalService {
     }
 
     const response = await fetch(
-      `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${key}`,
+      `https://generativelanguage.googleapis.com/v1beta/models/${await this.getVisionModel("gemini", "gemini-2.0-flash")}:generateContent?key=${key}`,
       {
         method: "POST",
         headers: { "Content-Type": "application/json" },
@@ -229,7 +237,7 @@ class MultimodalService {
         "anthropic-dangerous-direct-browser-access": "true",
       },
       body: JSON.stringify({
-        model: "claude-haiku-4-5-20251001",
+        model: await this.getVisionModel("anthropic", "claude-haiku-4-5-20251001"),
         max_tokens: 1000,
         messages: [{
           role: "user",
@@ -251,28 +259,26 @@ class MultimodalService {
     return textBlock?.text || ""
   }
 
-  getConfiguredVisionProviders(): Record<string, boolean> {
-    const db = getDb()
-    const hasDbKey = (providerId: string): boolean => {
-      const row = db.query(
-        `SELECT api_key_encrypted FROM providers WHERE id = ? AND api_key_encrypted IS NOT NULL AND api_key_encrypted != ''`
-      ).get(providerId) as { api_key_encrypted: string } | undefined
-      return !!row
-    }
+  async getConfiguredVisionProviders(): Promise<Record<string, boolean>> {
+    const hasDbKey = async (providerId: string): Promise<boolean> => !!(await loadProviderApiKey(providerId))
+
+    const [openai, gemini, anthropic] = await Promise.all([
+      hasDbKey("openai"), hasDbKey("gemini"), hasDbKey("anthropic"),
+    ])
 
     return {
-      openai: hasDbKey("openai") || !!(process.env.OPENAI_API_KEY),
-      gemini: hasDbKey("gemini") || !!(process.env.GEMINI_API_KEY),
-      anthropic: hasDbKey("anthropic") || !!(process.env.ANTHROPIC_API_KEY),
+      openai: openai || !!(process.env.OPENAI_API_KEY),
+      gemini: gemini || !!(process.env.GEMINI_API_KEY),
+      anthropic: anthropic || !!(process.env.ANTHROPIC_API_KEY),
     }
   }
 
-  modelSupportsVision(providerId: string, modelId: string): boolean {
-    const db = getDb()
-    const model = db.query(`SELECT capabilities FROM models WHERE id = ? AND provider_id = ?`).get(modelId, providerId) as { capabilities: string } | undefined
-    if (!model?.capabilities) return false
+  async modelSupportsVision(providerId: string, modelId: string): Promise<boolean> {
+    const modelsCol = await col<ModelDoc>("models")
+    const entry = await modelsCol.get(modelId)
+    if (!entry || entry.doc.provider_id !== providerId || !entry.doc.capabilities) return false
     try {
-      const caps = JSON.parse(model.capabilities) as string[]
+      const caps = JSON.parse(entry.doc.capabilities) as string[]
       return caps.includes("vision")
     } catch {
       return false

package/packages/core/src/resilience/retry.ts

@@ -0,0 +1,95 @@
+/**
+ * Generic retry-with-backoff helper for transient failures (LLM provider
+ * calls, network/tool errors). Complements CircuitBreaker (circuit-breaker.ts),
+ * which sits one layer out and trips on sustained failure rates — this module
+ * only decides whether a single call should be retried and how long to wait.
+ */
+
+import { logger } from "../utils/logger";
+
+const log = logger.child("retry");
+
+export interface RetryPolicy {
+  maxAttempts: number;
+  initialDelayMs: number;
+  backoffMultiplier: number;
+  maxDelayMs: number;
+}
+
+const DEFAULT_RETRY_POLICY: RetryPolicy = {
+  maxAttempts: 3,
+  initialDelayMs: 1000,
+  backoffMultiplier: 2,
+  maxDelayMs: 30_000,
+};
+
+/** Exponential backoff with full jitter, capped at policy.maxDelayMs. */
+export function computeRetryDelay(attempt: number, policy: RetryPolicy, retryAfterMs?: number): number {
+  if (retryAfterMs !== undefined) return Math.min(retryAfterMs, policy.maxDelayMs);
+  const base = Math.min(policy.maxDelayMs, policy.initialDelayMs * Math.pow(policy.backoffMultiplier, attempt));
+  return Math.round(base * (0.5 + Math.random() * 0.5));
+}
+
+/**
+ * HTTP-status-aware transient classifier: 408/429/5xx/network/timeout are
+ * retryable, 400/401/403/404 (and anything else) are terminal. Looks for a
+ * `status`/`statusCode` property (most provider SDKs set one) or common
+ * network/timeout error names/messages as a fallback.
+ */
+export function isRetryableError(err: unknown): boolean {
+  const status = extractStatus(err);
+  if (status !== undefined) {
+    if (status === 408 || status === 429) return true;
+    if (status >= 500 && status < 600) return true;
+    if (status >= 400 && status < 500) return false;
+  }
+  const name = (err as { name?: string })?.name ?? "";
+  const message = (err as { message?: string })?.message ?? String(err);
+  if (name === "AbortError") return false;
+  if (/timeout|timed out|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|network|fetch failed/i.test(`${name} ${message}`)) {
+    return true;
+  }
+  return false;
+}
+
+function extractStatus(err: unknown): number | undefined {
+  const e = err as { status?: number; statusCode?: number; response?: { status?: number } };
+  return e?.status ?? e?.statusCode ?? e?.response?.status;
+}
+
+/** Extract a `Retry-After` value (seconds or HTTP-date) if the error/response carries one. */
+function extractRetryAfterMs(err: unknown): number | undefined {
+  const headers = (err as { headers?: Record<string, string>; response?: { headers?: Record<string, string> } })?.headers
+    ?? (err as { response?: { headers?: Record<string, string> } })?.response?.headers;
+  const raw = headers?.["retry-after"] ?? headers?.["Retry-After"];
+  if (!raw) return undefined;
+  const seconds = Number(raw);
+  if (!Number.isNaN(seconds)) return seconds * 1000;
+  const dateMs = Date.parse(raw);
+  return Number.isNaN(dateMs) ? undefined : Math.max(0, dateMs - Date.now());
+}
+
+/**
+ * Run `fn`, retrying on transient failure per `policy`. `isRetryable`
+ * defaults to {@link isRetryableError}; pass a custom classifier for
+ * domain-specific errors (e.g. job executors).
+ */
+export async function withRetry<T>(
+  fn: (attempt: number) => Promise<T>,
+  policy: RetryPolicy = DEFAULT_RETRY_POLICY,
+  isRetryable: (err: unknown) => boolean = isRetryableError
+): Promise<T> {
+  let lastErr: unknown;
+  for (let attempt = 0; attempt < policy.maxAttempts; attempt++) {
+    try {
+      return await fn(attempt);
+    } catch (err) {
+      lastErr = err;
+      if (attempt === policy.maxAttempts - 1 || !isRetryable(err)) throw err;
+      const delay = computeRetryDelay(attempt, policy, extractRetryAfterMs(err));
+      log.warn(`[withRetry] Attempt ${attempt + 1}/${policy.maxAttempts} failed (${(err as Error).message}); retrying in ${delay}ms`);
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+  throw lastErr;
+}