@johpaz/hive-agents 0.0.35 → 0.0.37

package/packages/core/src/agent/llm-providers/openai-compat-base.ts

@@ -0,0 +1,379 @@
+import { logger } from "../../utils/logger"
+import {
+  sanitizeMessages, requiresTemperature1, OPENAI_COMPAT_BASE_URLS,
+  getProviderProfile, modelSupportsTools, normalizeToolName, normalizeToolSchema,
+} from "./interface"
+import type { LLMCallOptions, LLMProvider, LLMResponse, LLMToolCall } from "./interface"
+import type { ContentPart, LLMMessage } from "../llm-client"
+
+const log = logger.child("llm-client")
+
+export abstract class OpenAICompatBase implements LLMProvider {
+  constructor(protected readonly providerName: string) {}
+
+  /** Override to true when the provider requires reasoning_content to be round-tripped. */
+  protected needsReasoningRoundtrip(): boolean { return false }
+
+  /** Override to true for providers running on localhost. */
+  protected isLocalProvider(): boolean { return false }
+
+  /** Hook called before each request. Override for e.g. auto-starting a local server. */
+  protected async beforeCall(_options: LLMCallOptions): Promise<void> {}
+
+  /**
+   * Hook called after tools are prepared when sendTools is true.
+   * Override to inject tool descriptions into the system prompt
+   * (for local models whose chat templates don't support native tool calling).
+   */
+  protected injectToolsIntoPrompt(_body: any, _preparedTools: any[]): void {}
+
+  private _convertContentPart(part: ContentPart): any {
+    switch (part.type) {
+      case "text":
+        return { type: "text", text: part.text }
+      case "image_url":
+        return { type: "image_url", image_url: { url: part.image_url.url } }
+      case "image_base64":
+        return { type: "image_url", image_url: { url: `data:${part.mimeType};base64,${part.base64}` } }
+      case "document":
+        log.warn(`[llm-client] ${this.providerName}: document content parts are not supported — content will be omitted`)
+        return { type: "text", text: `[Document: ${(part as any).fileName || "file"}] (content not supported for this provider)` }
+      default:
+        return { type: "text", text: JSON.stringify(part) }
+    }
+  }
+
+  private _convertMessage(msg: LLMMessage): any {
+    if (Array.isArray(msg.content)) {
+      return { ...msg, content: msg.content.map(p => this._convertContentPart(p)) }
+    }
+    return msg
+  }
+
+  async call(options: LLMCallOptions): Promise<LLMResponse> {
+    const { default: OpenAI } = await import("openai")
+
+    const baseURL = options.baseUrl?.trim() || OPENAI_COMPAT_BASE_URLS[this.providerName] || undefined
+    const isLocal = this.isLocalProvider()
+
+    await this.beforeCall(options)
+
+    const apiKey = options.apiKey || (isLocal ? "ollama" : undefined)
+    if (!apiKey) {
+      throw new Error(`API key missing for provider: ${this.providerName}. Configure it in Settings → Providers.`)
+    }
+
+    const client = new OpenAI({ apiKey, baseURL })
+
+    const sanitized = sanitizeMessages(options.messages)
+    const rawMessages = this.needsReasoningRoundtrip()
+      ? sanitized
+      : sanitized.map(({ reasoning_content: _rc, ...rest }) => rest as typeof sanitized[number])
+    const messagesForProvider = rawMessages.map(m => this._convertMessage(m))
+
+    const providerPrefix = new RegExp(`^${this.providerName}\\/`, "i")
+    const body: any = {
+      model: options.model.replace(providerPrefix, ""),
+      messages: messagesForProvider,
+      temperature: requiresTemperature1(this.providerName, options.model) ? 1 : (options.temperature ?? 0.7),
+    }
+    if (options.maxTokens) {
+      body.max_tokens = options.maxTokens
+    } else if (options.contextWindow) {
+      // Reserve 15% of context window for output tokens, cap at 8192
+      body.max_tokens = Math.min(8192, Math.floor(options.contextWindow * 0.15))
+    }
+    if (options.numCtx && isLocal) body.num_ctx = options.numCtx
+
+    const profile = getProviderProfile(this.providerName)
+    const sendTools = modelSupportsTools(this.providerName, options.model) && !!(options.tools?.length)
+
+    const toolNameMap = new Map<string, string>()
+
+    if (sendTools) {
+      const preparedTools = options.tools!.map((t) => {
+        const originalName = t.function.name
+        const wireName = profile.normalizeToolNames
+          ? normalizeToolName(originalName, profile.toolNameReplacement)
+          : originalName
+        if (wireName !== originalName) toolNameMap.set(wireName, originalName)
+        return {
+          ...t,
+          function: {
+            ...t.function,
+            name: wireName,
+            parameters: normalizeToolSchema(t.function.parameters as Record<string, unknown>, profile),
+          },
+        }
+      })
+      body.tools = preparedTools
+      body.tool_choice = profile.toolChoiceAuto
+      if (profile.disableParallelToolCalls) body.parallel_tool_calls = false
+
+      this.injectToolsIntoPrompt(body, preparedTools)
+    }
+
+    log.info(`[llm-client] ${this.providerName}/${body.model} — ${options.messages.length} msgs, ${options.tools?.length ?? 0} tools${sendTools ? "" : " (tools suppressed)"}`)
+
+    if (options.onToken) {
+      return this._streamCall(client, body, options, toolNameMap, sendTools, profile)
+    }
+
+    let response
+    try {
+      response = await client.chat.completions.create(body)
+    } catch (err: any) {
+      const status = err?.status ?? err?.response?.status
+      const errMsg = (err?.error?.message ?? err?.message ?? "").toLowerCase()
+
+      // Retry 1: tools rejected by provider — remove tools and retry
+      if (sendTools && profile.retryWithoutToolsOnCodes.includes(status)) {
+        log.warn(`[llm-client] ${this.providerName}: tools rejected (HTTP ${status}) — retrying without tools`)
+        const bodyNoTools = { ...body }
+        delete bodyNoTools.tools
+        delete bodyNoTools.tool_choice
+        delete bodyNoTools.parallel_tool_calls
+        response = await client.chat.completions.create(bodyNoTools)
+      }
+      // Retry 2: context overflow — compact messages and retry
+      else if (status === 400 && (errMsg.includes("context length") || errMsg.includes("input_tokens") || errMsg.includes("maximum input length"))) {
+        log.warn(`[llm-client] ${this.providerName}: context overflow — compacting messages and retrying`)
+        const compacted = [...body.messages]
+        const kept: any[] = []
+        let systemMsg: any = null
+        for (const m of compacted) {
+          if (m.role === "system") { systemMsg = m; continue }
+          kept.push(m)
+        }
+        // Keep only 33% of messages to free enough context budget
+        const keepRatio = Math.max(1, Math.floor(kept.length / 3))
+        const trimmed = kept.slice(-keepRatio)
+        body.messages = systemMsg ? [systemMsg, ...trimmed] : trimmed
+        // Reduce output budget to leave more room for input
+        if (body.max_tokens) body.max_tokens = Math.min(body.max_tokens, 4096)
+
+        log.info(`[llm-client] ${this.providerName}: compacted ${compacted.length} msgs → ${body.messages.length} msgs, max_tokens=${body.max_tokens}`)
+        response = await client.chat.completions.create(body)
+      }
+      else {
+        throw err
+      }
+    }
+
+    const choice = response.choices[0]
+    const msg = choice.message
+
+    let final_tool_calls: LLMToolCall[] | undefined = (msg.tool_calls as any[])?.map((tc: any) => ({
+      id: tc.id,
+      type: "function" as const,
+      function: {
+        name: toolNameMap.get(tc.function.name) ?? tc.function.name,
+        arguments: tc.function.arguments,
+      },
+    }))
+
+    let final_content = msg.content ?? ""
+
+    if (sendTools && (!final_tool_calls || final_tool_calls.length === 0) && final_content) {
+      const extracted = extractToolCallsFromText(final_content, toolNameMap)
+      if (extracted.tool_calls.length > 0) {
+        final_tool_calls = extracted.tool_calls
+        final_content = extracted.content
+      }
+    }
+
+    return {
+      content: final_content,
+      tool_calls: final_tool_calls,
+      reasoning_content: (msg as any).reasoning_content ?? undefined,
+      stop_reason:
+        choice.finish_reason === "tool_calls" ? "tool_calls"
+          : choice.finish_reason === "length" ? "max_tokens"
+            : "stop",
+      usage: response.usage ? {
+        input_tokens: response.usage.prompt_tokens,
+        output_tokens: response.usage.completion_tokens,
+      } : undefined,
+    }
+  }
+
+  private async _streamCall(
+    client: any,
+    body: any,
+    options: LLMCallOptions,
+    toolNameMap: Map<string, string>,
+    sendTools: boolean,
+    profile: ReturnType<typeof getProviderProfile>,
+  ): Promise<LLMResponse> {
+    let stream
+    try {
+      stream = await client.chat.completions.create({ ...body, stream: true })
+    } catch (err: any) {
+      const status = err?.status ?? err?.response?.status
+      const errMsg = (err?.error?.message ?? err?.message ?? "").toLowerCase()
+
+      if (sendTools && profile.retryWithoutToolsOnCodes.includes(status)) {
+        log.warn(`[llm-client] ${this.providerName}: tools rejected (HTTP ${status}) — retrying stream without tools`)
+        const bodyNoTools = { ...body }
+        delete bodyNoTools.tools
+        delete bodyNoTools.tool_choice
+        delete bodyNoTools.parallel_tool_calls
+        stream = await client.chat.completions.create({ ...bodyNoTools, stream: true })
+      } else if (status === 400 && (errMsg.includes("context length") || errMsg.includes("input_tokens") || errMsg.includes("maximum input length"))) {
+        log.warn(`[llm-client] ${this.providerName}: context overflow — compacting messages and retrying stream`)
+        const compacted = [...body.messages]
+        const kept: any[] = []
+        let systemMsg: any = null
+        for (const m of compacted) {
+          if (m.role === "system") { systemMsg = m; continue }
+          kept.push(m)
+        }
+        const keepRatio = Math.max(1, Math.floor(kept.length / 3))
+        const trimmed = kept.slice(-keepRatio)
+        body.messages = systemMsg ? [systemMsg, ...trimmed] : trimmed
+        if (body.max_tokens) body.max_tokens = Math.min(body.max_tokens, 4096)
+        log.info(`[llm-client] ${this.providerName}: compacted ${compacted.length} msgs → ${body.messages.length} msgs, max_tokens=${body.max_tokens}`)
+        stream = await client.chat.completions.create({ ...body, stream: true })
+      } else {
+        throw err
+      }
+    }
+
+    let content = ""
+    let reasoning_content = ""
+    let finish_reason = "stop"
+    const toolCallMap: Map<number, { id: string; name: string; arguments: string }> = new Map()
+    let input_tokens = 0
+    let output_tokens = 0
+
+    for await (const chunk of stream) {
+      const choice = chunk.choices?.[0]
+      if (!choice) continue
+
+      const delta = choice.delta as any
+      if (delta.content) {
+        content += delta.content
+        options.onToken!(delta.content)
+      }
+      if (delta.reasoning_content) {
+        reasoning_content += delta.reasoning_content
+      }
+      if (delta.tool_calls) {
+        for (const tc of delta.tool_calls) {
+          const idx: number = tc.index
+          if (!toolCallMap.has(idx)) {
+            toolCallMap.set(idx, { id: tc.id ?? "", name: tc.function?.name ?? "", arguments: "" })
+          }
+          const entry = toolCallMap.get(idx)!
+          if (tc.id) entry.id = tc.id
+          if (tc.function?.name) entry.name = tc.function.name
+          if (tc.function?.arguments) entry.arguments += tc.function.arguments
+        }
+      }
+      if (choice.finish_reason) finish_reason = choice.finish_reason
+
+      if (chunk.usage) {
+        input_tokens = chunk.usage.prompt_tokens ?? 0
+        output_tokens = chunk.usage.completion_tokens ?? 0
+      }
+    }
+
+    const tool_calls: LLMToolCall[] = [...toolCallMap.values()].map((tc) => ({
+      id: tc.id,
+      type: "function" as const,
+      function: {
+        name: toolNameMap.get(tc.name) ?? tc.name,
+        arguments: tc.arguments || "{}",
+      },
+    }))
+
+    let final_tool_calls: LLMToolCall[] | undefined = tool_calls.length ? tool_calls : undefined
+    let final_content = content
+
+    if (sendTools && !final_tool_calls && final_content) {
+      const extracted = extractToolCallsFromText(final_content, toolNameMap)
+      if (extracted.tool_calls.length > 0) {
+        final_tool_calls = extracted.tool_calls
+        final_content = extracted.content
+      }
+    }
+
+    return {
+      content: final_content,
+      tool_calls: final_tool_calls,
+      reasoning_content: reasoning_content || undefined,
+      stop_reason:
+        finish_reason === "tool_calls" ? "tool_calls"
+          : finish_reason === "length" ? "max_tokens"
+            : "stop",
+      usage: input_tokens > 0 || output_tokens > 0
+        ? { input_tokens, output_tokens }
+        : undefined,
+    }
+  }
+}
+
+/**
+ * Extracts tool_calls from text when the model fails to emit native tool_calls.
+ * Supports common formats used by Gemma, Qwen, and other local models.
+ */
+export function extractToolCallsFromText(
+  content: string,
+  toolNameMap: Map<string, string>,
+  knownToolNames?: Set<string>,
+): { content: string; tool_calls: LLMToolCall[] } {
+  const tool_calls: LLMToolCall[] = []
+  let extractedContent = content
+
+  const regexes = [
+    /<tool_call>\s*({[\s\S]*?})\s*<\/tool_call>/g,
+    /<function_call>\s*({[\s\S]*?})\s*<\/function_call>/g,
+    /```(?:tool_call|json)\s*({[\s\S]*?})\s*```/g,
+  ]
+
+  for (const regex of regexes) {
+    let match
+    while ((match = regex.exec(content)) !== null) {
+      try {
+        const json = JSON.parse(match[1])
+        if (json.name) {
+          tool_calls.push({
+            id: crypto.randomUUID(),
+            type: "function",
+            function: {
+              name: toolNameMap.get(json.name) ?? json.name,
+              arguments: typeof json.arguments === "object" ? JSON.stringify(json.arguments) : (json.arguments || "{}"),
+            },
+          })
+          extractedContent = extractedContent.replace(match[0], "").trim()
+        }
+      } catch {
+        // ignore parse errors
+      }
+    }
+  }
+
+  // Fallback: entire output is a bare JSON tool call — only if name matches a known tool
+  if (tool_calls.length === 0 && knownToolNames && knownToolNames.size > 0) {
+    try {
+      const json = JSON.parse(content.trim())
+      const resolvedName = toolNameMap.get(json.name) ?? json.name
+      if (json.name && knownToolNames.has(resolvedName) && (json.arguments || json.parameters)) {
+        const args = json.arguments || json.parameters
+        tool_calls.push({
+          id: crypto.randomUUID(),
+          type: "function",
+          function: {
+            name: resolvedName,
+            arguments: typeof args === "object" ? JSON.stringify(args) : (args || "{}"),
+          },
+        })
+        extractedContent = ""
+      }
+    } catch {
+      // not valid JSON
+    }
+  }
+
+  return { content: extractedContent, tool_calls }
+}

package/packages/core/src/agent/llm-providers/openai.ts

@@ -0,0 +1,5 @@
+import { OpenAICompatBase } from "./openai-compat-base"
+
+export class OpenAIProvider extends OpenAICompatBase {
+  constructor() { super("openai") }
+}

package/packages/core/src/agent/llm-providers/openrouter.ts

@@ -0,0 +1,5 @@
+import { OpenAICompatBase } from "./openai-compat-base"
+
+export class OpenRouterProvider extends OpenAICompatBase {
+  constructor() { super("openrouter") }
+}

package/packages/core/src/agent/llm-providers/qwen.ts

@@ -0,0 +1,5 @@
+import { OpenAICompatBase } from "./openai-compat-base"
+
+export class QwenProvider extends OpenAICompatBase {
+  constructor() { super("qwen") }
+}

package/packages/core/src/agent/native-tools.ts

@@ -0,0 +1,31 @@
+/**
+ * Native Tools Type Definitions
+ * 
+ * Tool interface for native Hive tools (no LangChain)
+ */
+
+export interface Tool {
+  name: string
+  description: string
+  parameters: {
+    type: "object"
+    properties: Record<string, ToolParameter>
+    required?: string[]
+  }
+  execute: (params: Record<string, unknown>, config?: any) => Promise<string | object>
+}
+
+export interface ToolParameter {
+  type: string
+  description?: string
+  enum?: string[]
+  items?: ToolParameter
+  properties?: Record<string, ToolParameter>
+  required?: string[]
+}
+
+export interface ToolResult {
+  success: boolean
+  result?: any
+  error?: string
+}

package/packages/core/src/agent/playbook-selector.ts

@@ -0,0 +1,147 @@
+/**
+ * FTS5-based Playbook Rules Selector (ACE Curator)
+ * 
+ * This module allows the Context Compiler to inject relevant evolved rules
+ * into the agent prompt based on semantic relevance to the current message.
+ */
+
+import { getDb } from "../storage/sqlite"
+import { logger } from "../utils/logger"
+
+const log = logger.child("playbook-selector")
+
+// ─── Types ───────────────────────────────────────────────────────────────────────
+
+export interface PlaybookRule {
+    id: number
+    rule: string
+    category: string
+    applicable_to?: string
+}
+
+// ─── Configuration ─────────────────────────────────────────────────────────────
+
+/** Maximum rules to inject per context window */
+const MAX_RULES_PER_TURN = 5
+
+/** Minimum bm25 score threshold for rules */
+const MIN_RELEVANCE_THRESHOLD = -10 // Relaxed for better matching
+
+// ─── Selection Logic ───────────────────────────────────────────────────────────
+
+/**
+ * Select relevant rules from the Playbook based on semantic matching
+ */
+export function selectPlaybookRules(message: string): PlaybookRule[] {
+    const db = getDb()
+    const startTime = performance.now()
+
+    // Clean query — use prefix matching for consistency with skill-selector and tool-selector
+    const keywords = message
+        .toLowerCase()
+        // Keep only letters, numbers, spaces (strips ALL FTS5 special syntax)
+        .replace(/[^\p{L}\p{N}\s]/gu, " ")
+        .split(/\s+/)
+        .filter(w => w.length > 3)
+        .slice(0, 5)
+
+    if (keywords.length === 0) return []
+
+    // Use prefix matching for better recall (e.g., "program*" matches "programar", "programación")
+    const ftsQuery = keywords.map(w => `${w}*`).join(" OR ")
+
+    try {
+        // Query FTS table
+        const ftsResults = db.query(`
+            SELECT rowid, bm25(playbook_fts) as score
+            FROM playbook_fts
+            WHERE playbook_fts MATCH ?
+            ORDER BY score ASC
+            LIMIT ?
+        `).all(ftsQuery, MAX_RULES_PER_TURN) as Array<{ rowid: number; score: number }>
+
+        const relevantIds = ftsResults
+            .filter(r => r.score >= MIN_RELEVANCE_THRESHOLD)
+            .map(r => r.rowid)
+
+        if (relevantIds.length === 0) return []
+
+        // Fetch full rules
+        const rules = db.query(`
+            SELECT id, rule, category, applicable_to
+            FROM playbook
+            WHERE id IN (${relevantIds.map(() => '?').join(',')})
+            AND active = 1
+        `).all(...relevantIds) as PlaybookRule[]
+
+        const timing = performance.now() - startTime
+        log.info(`[playbook-selector] Selected ${rules.length} rules in ${timing.toFixed(2)}ms`)
+        if (rules.length > 0) {
+          log.debug(`[playbook-selector] Rules: ${rules.map(r => `[${r.id}] ${r.rule.substring(0, 60)}`).join(', ')}`)
+        }
+
+        return rules
+    } catch (err) {
+        log.error(`[playbook-selector] Failed to select rules:`, err)
+        return []
+    }
+}
+
+// ─── Sync Logic ───────────────────────────────────────────────────────────────
+
+/**
+ * Sync active playbook rules to FTS5 virtual table
+ */
+export async function syncPlaybookToFTS(): Promise<void> {
+    const db = getDb()
+
+    try {
+        // Step 1: Get active rules
+        const rules = db.query(`
+            SELECT id, rule, category, applicable_to
+            FROM playbook
+            WHERE active = 1
+        `).all() as Array<{
+            id: number
+            rule: string
+            category: string
+            applicable_to: string
+        }>
+
+        if (rules.length === 0) {
+            log.debug(`[playbook-selector] No rules in playbook to sync`)
+        }
+
+        // Step 2: Atomic transaction for FTS5 sync
+        const syncTransaction = db.transaction(() => {
+            // Verify table exists
+            const tableCheck = db.query("SELECT name FROM sqlite_master WHERE type='table' AND name='playbook_fts'").get()
+            if (!tableCheck) {
+                throw new Error("playbook_fts table does not exist!")
+            }
+
+            // A: Clear existing data
+            db.run("DELETE FROM playbook_fts")
+
+            // B: Prepare insertion
+            const insert = db.prepare(`
+                INSERT INTO playbook_fts(rowid, rule, category, applicable_to)
+                VALUES (?, ?, ?, ?)
+            `)
+
+            // C: Re-populate
+            for (const item of rules) {
+                insert.run(item.id, item.rule, item.category, item.applicable_to)
+            }
+        })
+
+        // Execute transaction
+        syncTransaction()
+
+        log.info(`[playbook-selector] Atomic sync complete: ${rules.length} rules indexed in FTS5`)
+
+    } catch (err) {
+        log.error(`[playbook-selector] Transactional sync failed:`, err)
+        throw err
+    }
+}