@johpaz/hive-sdk 0.1.3 → 0.1.5

package/packages/core/src/agent/capability-search.ts

@@ -0,0 +1,186 @@
+/**
+ * Capability Search — shared HiveDB search layer
+ *
+ * Single entry point for searching Hive's capability catalogs (native tools,
+ * skills, playbook rules, MCP tools): one HiveDB index for all four, with the
+ * catalog discriminated by the `type` field rather than by separate tables.
+ *
+ * Document convention (one index, type discrimination via filters):
+ * - id:   `tool:${name}` | `skill:${id}` | `playbook:${rowid}` | `mcp:${id}` | `agent:${id}`
+ * - name: tool/skill name or rule head        (BM25 boost 4.0)
+ * - tags: category + triggers + keywords      (BM25 boost 3.0)
+ * - body: description / rule text / content   (BM25 boost 2.0)
+ * - filters: { type: "tool"|"skill"|"playbook"|"mcp" }
+ *            plus { server_id } on MCP docs for per-server hot-reload.
+ *
+ * Score semantics: text-only queries return raw BM25 (positive, higher is
+ * better). Never compare against an absolute floor — BM25 magnitude depends
+ * on corpus and document length. Use applyRelativeCutoff() instead.
+ */
+
+import type { IndexDoc } from "@johpaz/hive-db";
+import { getHiveDb } from "../storage/hivedb";
+import { logger } from "../utils/logger";
+
+const log = logger.child("capability-search");
+
+export type CapabilityType = "tool" | "skill" | "playbook" | "mcp" | "agent";
+
+export interface CapabilityHit {
+  /** Namespaced id, e.g. "tool:web_search" */
+  id: string;
+  type: CapabilityType;
+  /** Id without the type prefix, e.g. "web_search" */
+  rawId: string;
+  /** Raw BM25 score: positive, higher = more relevant */
+  score: number;
+}
+
+export interface CapabilityDoc {
+  type: CapabilityType;
+  /** Id within the type namespace (tool name, skill id, playbook rowid, mcp id) */
+  rawId: string;
+  name?: string;
+  body?: string;
+  tags?: string;
+  /** Extra filters besides `type` (e.g. { server_id } for MCP docs) */
+  extraFilters?: Array<{ field: string; value: string }>;
+}
+
+const TYPE_PREFIXES: CapabilityType[] = ["tool", "skill", "playbook", "mcp", "agent"];
+
+function splitId(id: string): { type: CapabilityType; rawId: string } | null {
+  const sep = id.indexOf(":");
+  if (sep === -1) return null;
+  const type = id.slice(0, sep) as CapabilityType;
+  if (!TYPE_PREFIXES.includes(type)) return null;
+  return { type, rawId: id.slice(sep + 1) };
+}
+
+// ─── Search ──────────────────────────────────────────────────────────────────
+
+export interface SearchCapabilitiesOptions {
+  /** Restrict to these types. Omit or empty = all types. */
+  types?: CapabilityType[];
+  /** Maximum hits to return (default 10). */
+  k?: number;
+  /** Per-field BM25 boosts (engine defaults: name 4, tags 3, body 2). */
+  boosts?: { name?: number; body?: number; tags?: number };
+}
+
+/**
+ * Search the capability index with raw user text. The engine parses leniently
+ * (accents, quotes, operators and punctuation never throw) and applies
+ * Spanish stemming + accent folding, so callers must NOT pre-sanitize.
+ */
+export async function searchCapabilities(
+  query: string,
+  opts: SearchCapabilitiesOptions = {}
+): Promise<CapabilityHit[]> {
+  const k = opts.k ?? 10;
+  const types = opts.types?.length ? opts.types : undefined;
+  const trimmed = query.trim();
+  if (!trimmed) return [];
+
+  const startTime = performance.now();
+  const db = await getHiveDb();
+
+  // Filters are AND-ed by the engine, so multi-type search runs one query per
+  // type; the single-type and all-types cases are one engine call.
+  const queries = types
+    ? types.map((t) => ({
+        type: t,
+        filters: [{ field: "type", value: t }],
+      }))
+    : [{ type: undefined, filters: undefined }];
+
+  const merged = new Map<string, CapabilityHit>();
+  for (const q of queries) {
+    const hits = await db.queryHybrid({
+      text: trimmed,
+      k,
+      filters: q.filters,
+      boosts: opts.boosts,
+    });
+    for (const hit of hits) {
+      const parsed = splitId(hit.id);
+      if (!parsed) continue;
+      const existing = merged.get(hit.id);
+      if (!existing || hit.score > existing.score) {
+        merged.set(hit.id, {
+          id: hit.id,
+          type: parsed.type,
+          rawId: parsed.rawId,
+          score: hit.score,
+        });
+      }
+    }
+  }
+
+  const results = Array.from(merged.values())
+    .sort((a, b) => b.score - a.score)
+    .slice(0, k);
+
+  const timing = performance.now() - startTime;
+  log.debug(
+    `[capability-search] "${trimmed.substring(0, 60)}" → ${results.length} hits in ${timing.toFixed(1)}ms`
+  );
+  return results;
+}
+
+/**
+ * Keep only hits scoring at least `ratio` of the top hit. This replaces the
+ * old absolute negative-bm25 thresholds: relevance is relative to the best
+ * match, never an absolute floor.
+ */
+export function applyRelativeCutoff(
+  hits: CapabilityHit[],
+  ratio = 0.3
+): CapabilityHit[] {
+  if (hits.length === 0) return hits;
+  const top = hits[0].score;
+  if (top <= 0) return [];
+  return hits.filter((h) => h.score >= ratio * top);
+}
+
+// ─── Sync helpers ────────────────────────────────────────────────────────────
+
+/**
+ * Replace all documents of a type: deletes existing docs carrying the type
+ * filter, then batch-upserts the new set under a single index commit.
+ */
+export async function replaceCapabilityDocs(
+  type: CapabilityType,
+  docs: CapabilityDoc[]
+): Promise<void> {
+  const db = await getHiveDb();
+  await db.deleteByFilter({ field: "type", value: type });
+  if (docs.length === 0) return;
+  await db.upsertBatch(docs.map(toIndexDoc));
+}
+
+/** Upsert documents without clearing the rest of their type. */
+export async function upsertCapabilityDocs(docs: CapabilityDoc[]): Promise<void> {
+  if (docs.length === 0) return;
+  const db = await getHiveDb();
+  await db.upsertBatch(docs.map(toIndexDoc));
+}
+
+/** Delete every MCP doc belonging to a server (hot-reload/disconnect). */
+export async function deleteCapabilitiesByServer(serverId: string): Promise<void> {
+  const db = await getHiveDb();
+  await db.deleteByFilter({ field: "server_id", value: serverId });
+}
+
+function toIndexDoc(doc: CapabilityDoc): IndexDoc {
+  return {
+    id: `${doc.type}:${doc.rawId}`,
+    name: doc.name,
+    body: doc.body,
+    tags: doc.tags,
+    filters: [
+      { field: "type", value: doc.type },
+      ...(doc.extraFilters ?? []),
+    ],
+  };
+}

package/packages/core/src/agent/catalog-selector.ts

@@ -0,0 +1,103 @@
+import { col } from "../storage/hive";
+import type { AgentDoc } from "../storage/collections";
+import {
+  applyRelativeCutoff,
+  replaceCapabilityDocs,
+  searchCapabilities,
+  type CapabilityDoc,
+} from "./capability-search";
+import { logger } from "../utils/logger";
+
+const log = logger.child("catalog-selector");
+const RELEVANCE_RATIO = 0.35;
+const ROUTING_STOP_WORDS = new Set([
+  "a", "al", "de", "del", "el", "en", "la", "las", "lo", "los",
+  "o", "otro", "otra", "un", "una", "y",
+]);
+
+function routingTokens(value: string): Set<string> {
+  const normalized = value
+    .normalize("NFD")
+    .replace(/\p{Diacritic}/gu, "")
+    .toLowerCase()
+    .split(/[^\p{Letter}\p{Number}_]+/u)
+    .filter(Boolean)
+    .map((token) => token.length > 4 && token.endsWith("s") ? token.slice(0, -1) : token)
+    .filter((token) => !ROUTING_STOP_WORDS.has(token));
+  return new Set(normalized);
+}
+
+function matchesRoutingExclusion(query: string, agent: AgentDoc): boolean {
+  if (!agent.routing_exclusions_json) return false;
+  const queryTokens = routingTokens(query);
+  const exclusions = JSON.parse(agent.routing_exclusions_json) as string[];
+
+  return exclusions.some((exclusion) => {
+    const excludedTokens = routingTokens(exclusion);
+    const shared = [...excludedTokens].filter((token) => queryTokens.has(token)).length;
+    return shared >= 2 && shared / Math.max(excludedTokens.size, 1) >= 0.35;
+  });
+}
+
+export async function getCatalogAgent(id: string): Promise<AgentDoc | null> {
+  const entry = await (await col<AgentDoc>("agents")).get(id);
+  return entry?.doc.source === "catalog" && entry.doc.enabled ? entry.doc : null;
+}
+
+export async function listCatalogAgents(): Promise<AgentDoc[]> {
+  const c = await col<AgentDoc>("agents");
+  return (await c.findBy("source", "catalog")).map((entry) => entry.doc).filter((doc) => doc.enabled);
+}
+
+export async function searchCatalogAgents(query: string, k = 5): Promise<Array<{ agent: AgentDoc; score: number }>> {
+  const hits = applyRelativeCutoff(
+    await searchCapabilities(query, { types: ["agent"], k }),
+    RELEVANCE_RATIO,
+  );
+  const c = await col<AgentDoc>("agents");
+  const results: Array<{ agent: AgentDoc; score: number }> = [];
+  for (const hit of hits) {
+    const entry = await c.get(hit.rawId);
+    if (
+      entry?.doc.source === "catalog" &&
+      entry.doc.enabled &&
+      !matchesRoutingExclusion(query, entry.doc)
+    ) {
+      results.push({ agent: entry.doc, score: hit.score });
+    }
+  }
+  return results;
+}
+
+export async function syncCatalogAgentsToIndex(): Promise<void> {
+  const agents = await listCatalogAgents();
+  const docs: CapabilityDoc[] = agents.map((agent) => ({
+    type: "agent",
+    rawId: agent.id,
+    name: `${agent.name} ${agent.id}`,
+    tags: [
+      ...(agent.routing_examples_json ? JSON.parse(agent.routing_examples_json) : []),
+      ...(agent.tool_allowlist_json ? JSON.parse(agent.tool_allowlist_json) : []),
+      ...(agent.skills_json ? JSON.parse(agent.skills_json) : []),
+    ].join(" "),
+    // The complete system prompt is intentionally excluded: it adds routing
+    // noise and may contain operational policy not meant for retrieval.
+    body: agent.description ?? "",
+  }));
+  await replaceCapabilityDocs("agent", docs);
+  log.info(`[catalog-selector] Synced ${docs.length} catalog agents to HiveDB index`);
+}
+
+export function renderAgentRoutingCatalog(agents: AgentDoc[]): string {
+  return agents
+    .map((a) => {
+      const exclusions = a.routing_exclusions_json
+        ? JSON.parse(a.routing_exclusions_json) as string[]
+        : [];
+      const exclusionText = exclusions.length > 0
+        ? ` NO usar para: ${exclusions.join("; ")}.`
+        : "";
+      return `- ${a.id}: ${a.description ?? ""}${exclusionText}`;
+    })
+    .join("\n");
+}

package/packages/core/src/agent/{Compaction.ts → compaction.ts}

@@ -15,23 +15,26 @@
  * short summaries in the in-memory message array before model calls.
  */
 
-import { logger } from "../utils/logger.ts"
+import { logger } from "../utils/logger"
 import {
   getTotalTokens,
   getHistory,
   getSummary,
   saveSummary,
-  toAPIMessages,
   getMessageCount,
-} from "./ConversationStore"
-import { estimateTokens } from "../utils/toon.ts"
-import { callLLM, resolveProviderConfig, type ContentPart } from "./providers/LLMClient"
-import { getDb } from "../storage/SQLiteStorage.ts"
+  isInternalSource,
+  type StoredMessage,
+} from "./conversation-store"
+import { estimateTokens } from "../utils/toon"
+import { callLLM, resolveProviderConfig, getDefaultLLM, type ContentPart } from "./llm-client"
+import { col, fromIndexable } from "../storage/hive"
+import type { AgentDoc, ModelDoc } from "../storage/collections"
 
 const log = logger.child("compaction")
 
 // Token budget: compress when stored tokens exceed this threshold
-const COMPACT_TOKEN_THRESHOLD = 6000   // ~60% of 10K context window
+// Will be overridden by model's context_window at runtime if available
+const COMPACT_TOKEN_THRESHOLD = 32000  // ~25% of 128K default context window
 const KEEP_LAST_N_MESSAGES = 5         // always keep most recent N messages
 const TOOL_RESULT_MAX_CHARS = 200      // max chars for old tool results after clearing
 const MAX_TRANSCRIPT_MSGS = 30         // cap messages sent to summarizer (avoids OOM on small models)
@@ -46,11 +49,30 @@ export async function maybeCompact(
   notify?: { channel: string; userId: string }
 ): Promise<void> {
   try {
-    const totalTokens = getTotalTokens(threadId)
-    if (totalTokens < COMPACT_TOKEN_THRESHOLD) return
+    const totalTokens = await getTotalTokens(threadId)
 
-    const summary = getSummary(threadId)
-    const totalMessages = getMessageCount(threadId)
+    // Use model's context window if available, otherwise use default
+    let effectiveThreshold = COMPACT_TOKEN_THRESHOLD
+    try {
+      const agentsCol = await col<AgentDoc>("agents")
+      const coordinators = await agentsCol.findBy("role", "coordinator", { limit: 1 })
+      const modelId = fromIndexable(coordinators[0]?.doc.model_id ?? null)
+      if (modelId) {
+        const modelsCol = await col<ModelDoc>("models")
+        // El id se busca completo: recortar el primer segmento rompía cualquier
+        // modelo cuyo nombre lleve barra (meta/llama-3.3-70b-instruct buscaba
+        // "llama-3.3-70b-instruct", no encontraba nada y caía al default).
+        const modelEntry = await modelsCol.get(modelId)
+        if (modelEntry?.doc.context_window) {
+          effectiveThreshold = Math.floor(modelEntry.doc.context_window * 0.25)
+        }
+      }
+    } catch { /* use default threshold */ }
+
+    if (totalTokens < effectiveThreshold) return
+
+    const summary = await getSummary(threadId)
+    const totalMessages = await getMessageCount(threadId)
 
     // Already summarized up to near the current state
     if (summary && summary.last_message_id > totalMessages - KEEP_LAST_N_MESSAGES) return
@@ -62,22 +84,48 @@ export async function maybeCompact(
   }
 }
 
+/**
+ * Find a clean cut point: the "keep" side must begin with a user turn so we
+ * never leave orphaned tool messages at the start of the visible window.
+ * Internal events (delegation fan-in) are persisted as role:"user", so they
+ * are valid boundaries here same as human turns.
+ * Returns 0 when no clean boundary exists (caller should skip compaction).
+ */
+export function findCompactionCutIndex(rows: StoredMessage[], keepLastN = KEEP_LAST_N_MESSAGES): number {
+  let cutIndex = rows.length - keepLastN
+  while (cutIndex > 0 && rows[cutIndex]?.role !== "user") {
+    cutIndex--
+  }
+  return cutIndex
+}
+
+/**
+ * Render a transcript for the summarizer LLM. Internal events (delegation
+ * fan-in notices) are labeled distinctly — they are persisted as
+ * role:"user" so the LLM treats them as input, but labeling them [USER] here
+ * would make the summarizer attribute system-generated task outcomes to the
+ * human, baking that misattribution into the durable summary.
+ */
+export function renderTranscript(rows: StoredMessage[], maxMsgChars = MAX_MSG_CHARS): string {
+  return rows
+    .map((r) => {
+      const label = isInternalSource(r.source) ? "EVENTO INTERNO" : r.role.toUpperCase()
+      return `[${label}]: ${r.content.substring(0, maxMsgChars)}`
+    })
+    .join("\n\n")
+}
+
 /**
  * Compress a thread's history into a summary.
  */
-export async function compactThread(
+async function compactThread(
   threadId: string,
   notify?: { channel: string; userId: string }
 ): Promise<void> {
-  const allMessages = getHistory(threadId)
+  const allMessages = await getHistory(threadId)
   if (allMessages.length <= KEEP_LAST_N_MESSAGES) return
 
-  // Find a clean cut point: the "keep" side must begin with a user turn so
-  // we never leave orphaned tool messages at the start of the visible window.
-  let cutIndex = allMessages.length - KEEP_LAST_N_MESSAGES
-  while (cutIndex > 0 && allMessages[cutIndex]?.role !== "user") {
-    cutIndex--
-  }
+  const cutIndex = findCompactionCutIndex(allMessages)
   if (cutIndex <= 0) {
     log.info(`[compaction] No clean user-turn boundary found — skipping`)
     return
@@ -88,32 +136,17 @@ export async function compactThread(
 
   const lastSummarizedId = toSummarize[toSummarize.length - 1].id
 
-  const existingSummary = getSummary(threadId)
+  const existingSummary = await getSummary(threadId)
   if (existingSummary && existingSummary.last_message_id >= lastSummarizedId) return
 
   // Cap transcript to avoid overflowing small model contexts
   const capped = toSummarize.slice(-MAX_TRANSCRIPT_MSGS)
-  const apiMessages = toAPIMessages(capped)
-  const transcript = apiMessages
-    .map((m) => {
-      const text = typeof m.content === "string"
-        ? m.content
-        : Array.isArray(m.content)
-          ? m.content.filter(p => p.type === "text").map(p => (p as any).text).join("\n")
-          : ""
-      return `[${m.role.toUpperCase()}]: ${text.substring(0, MAX_MSG_CHARS)}`
-    })
-    .join("\n\n")
+  const transcript = renderTranscript(capped)
 
-  const db = getDb()
-  const coordinator = db.query<any, []>(
-    "SELECT provider_id, model_id FROM agents WHERE role = 'coordinator' LIMIT 1"
-  ).get()
+  const defaultLLM = await getDefaultLLM()
+  if (!defaultLLM) throw new Error("No active LLM providers/models configured in the database")
 
-  const providerCfg = await resolveProviderConfig(
-    coordinator?.provider_id || "openai",
-    coordinator?.model_id || "gpt-4o-mini"
-  )
+  const providerCfg = await resolveProviderConfig(defaultLLM.provider, defaultLLM.model)
 
   const summaryResponse = await callLLM({
     ...providerCfg,
@@ -131,10 +164,22 @@ export async function compactThread(
     ],
   })
 
+  // A failed provider call still returns a populated `content` (the error text),
+  // so this has to gate on stop_reason — otherwise the summary that permanently
+  // replaces N messages of history becomes "[LLM Error] ...". Skipping leaves the
+  // thread uncompacted, which is recoverable; saving is not.
+  if (summaryResponse.stop_reason === "error") {
+    log.warn(
+      `[compaction] Summarizer call failed (${summaryResponse.error?.message ?? "unknown error"}) — `
+      + `keeping thread ${threadId} uncompacted rather than saving the error as its summary`
+    )
+    return
+  }
+
   const summary = summaryResponse.content.trim()
   if (!summary) return
 
-  saveSummary(threadId, summary, toSummarize.length, lastSummarizedId)
+  await saveSummary(threadId, summary, toSummarize.length, lastSummarizedId)
   log.info(
     `[compaction] Thread ${threadId} compacted: ${toSummarize.length} msgs → ${estimateTokens(summary)} tokens`
   )
@@ -142,7 +187,7 @@ export async function compactThread(
   // Notify user in their active channel (non-critical)
   if (notify?.channel && notify?.userId) {
     try {
-      const { sendToUserChannel } = await import("../gateway/channel-notify.ts")
+      const { sendToUserChannel } = await import("../gateway/channel-notify")
       await sendToUserChannel(
         notify.channel,
         notify.userId,
@@ -197,25 +242,3 @@ export function clearOldToolResults<T extends { role: string; content: string |
   })
 }
 
-/**
- * Summarize a tool result to a single line
- * Used for very old tool results (> 10 turns)
- */
-export function summarizeToolResult(content: string, toolName?: string): string {
-  // Try to extract success/failure status
-  const isError = content.includes('error') || content.includes('failed') || content.startsWith('[Tool Error]')
-  const isSuccess = content.includes('ok') || content.includes('success') || content.includes('true')
-  
-  // Try to extract key result field from JSON/TOON
-  let keyInfo = ""
-  try {
-    // Simple extraction of first key value
-    const firstLine = content.split('\n')[0].substring(0, 80)
-    keyInfo = firstLine
-  } catch {
-    keyInfo = content.substring(0, 80)
-  }
-  
-  const status = isError ? "failed" : isSuccess ? "success" : "completed"
-  return `[${toolName || 'Tool'} ${status}: ${keyInfo}...]`
-}