@johpaz/hive-agents 0.0.35 → 0.0.37

package/packages/core/src/agent/conversation-store.ts

@@ -0,0 +1,244 @@
+/**
+ * Conversation Store — persists message history in the `conversations` table.
+ * Replaces the LangGraph BunSqliteSaver + lg_checkpoints approach.
+ *
+ * Also manages: summaries, scratchpad.
+ */
+
+import { getDb } from "../storage/sqlite"
+import { logger } from "../utils/logger"
+import type { LLMMessage, ContentPart } from "./llm-client"
+import { estimateTokens } from "../utils/toon"
+
+const log = logger.child("conv-store")
+
+// ─── Types ────────────────────────────────────────────────────────────────────
+
+export interface StoredMessage {
+  id: number
+  thread_id: string
+  channel: string
+  role: "user" | "assistant" | "tool" | "system"
+  content: string
+  tool_calls_json: string | null
+  tool_call_id: string | null
+  reasoning_content: string | null  // Kimi K2 thinking — must be round-tripped
+  content_multimodal: string | null // JSON array of ContentPart[]
+  token_count: number
+  created_at: number
+}
+
+// ─── Message operations ───────────────────────────────────────────────────────
+
+export function addMessage(
+  threadId: string,
+  role: StoredMessage["role"],
+  content: string | ContentPart[],
+  opts?: {
+    channel?: string
+    tool_calls?: LLMMessage["tool_calls"]
+    tool_call_id?: string
+    reasoning_content?: string
+  }
+): number {
+  const db = getDb()
+  // Handle multimodal content by extracting text for the content column
+  const textContent = typeof content === "string"
+    ? content
+    : Array.isArray(content)
+      ? content.filter(p => p.type === "text").map(p => (p as any).text).join("\n")
+      : String(content)
+
+  const content_multimodal = Array.isArray(content) ? JSON.stringify(content) : null
+  const tool_calls_json = opts?.tool_calls ? JSON.stringify(opts.tool_calls) : null
+
+  const result = db.query(`
+    INSERT INTO conversations (thread_id, channel, role, content, content_multimodal, tool_calls_json, tool_call_id, reasoning_content, token_count, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, unixepoch())
+    RETURNING id
+  `).get(
+    threadId,
+    opts?.channel ?? "webchat",
+    role,
+    textContent,
+    content_multimodal,
+    tool_calls_json,
+    opts?.tool_call_id ?? null,
+    opts?.reasoning_content ?? null,
+    // Estimate tokens: content + tool_calls JSON
+    Math.max(1, estimateTokens(textContent) + estimateTokens(tool_calls_json ?? "")),
+  ) as { id: number }
+
+  return result.id
+}
+
+/**
+ * Returns all messages for the thread ordered oldest → newest.
+ */
+export function getHistory(threadId: string, limit = 200): StoredMessage[] {
+  const db = getDb()
+  return db.query(`
+    SELECT * FROM conversations
+    WHERE thread_id = ?
+    ORDER BY id ASC
+    LIMIT ?
+  `).all(threadId, limit) as StoredMessage[]
+}
+
+/**
+ * Returns only the last N messages (oldest → newest order),
+ * with leading orphaned tool messages stripped from the window start.
+ *
+ * A tool message is "orphaned" when the assistant message that issued its
+ * tool_call_id is not present in the loaded window (it was compacted away).
+ * Sending orphaned tool messages to the LLM causes provider errors.
+ */
+export function getRecentMessages(threadId: string, n: number): StoredMessage[] {
+  const db = getDb()
+  const rows = db.query(`
+    SELECT * FROM conversations
+    WHERE thread_id = ?
+    ORDER BY id DESC
+    LIMIT ?
+  `).all(threadId, n) as StoredMessage[]
+  return stripLeadingOrphanedTools(rows.reverse())
+}
+
+function stripLeadingOrphanedTools(rows: StoredMessage[]): StoredMessage[] {
+  // Collect all tool_call_ids referenced by assistant messages in this window
+  const knownIds = new Set<string>()
+  for (const r of rows) {
+    if (r.role === "assistant" && r.tool_calls_json) {
+      try {
+        const tcs = JSON.parse(r.tool_calls_json) as Array<{ id: string }>
+        for (const tc of tcs) knownIds.add(tc.id)
+      } catch { /* ignore malformed JSON */ }
+    }
+  }
+
+  // Drop tool messages at the start of the window whose assistant is missing
+  let start = 0
+  while (
+    start < rows.length &&
+    rows[start].role === "tool" &&
+    rows[start].tool_call_id !== null &&
+    !knownIds.has(rows[start].tool_call_id!)
+  ) {
+    start++
+  }
+
+  if (start > 0) {
+    log.warn(`[conv-store] Stripped ${start} leading orphaned tool message(s) from window (tool_call_ids outside window)`)
+  }
+  return start > 0 ? rows.slice(start) : rows
+}
+
+export function getMessageCount(threadId: string): number {
+  const db = getDb()
+  const row = db.query(
+    "SELECT COUNT(*) as cnt FROM conversations WHERE thread_id = ?"
+  ).get(threadId) as { cnt: number }
+  return row.cnt
+}
+
+export function getTotalTokens(threadId: string): number {
+  const db = getDb()
+  const row = db.query(
+    "SELECT COALESCE(SUM(token_count), 0) as total FROM conversations WHERE thread_id = ?"
+  ).get(threadId) as { total: number }
+  return row.total
+}
+
+/**
+ * Messages after a given message ID (for incremental summary updates).
+ */
+export function getMessagesAfter(threadId: string, afterId: number): StoredMessage[] {
+  const db = getDb()
+  return db.query(`
+    SELECT * FROM conversations
+    WHERE thread_id = ? AND id > ?
+    ORDER BY id ASC
+  `).all(threadId, afterId) as StoredMessage[]
+}
+
+// ─── Convert stored messages → LLMMessage array ───────────────────────────────
+
+export function toAPIMessages(rows: StoredMessage[]): LLMMessage[] {
+  return rows.map((r) => {
+    let content: string | ContentPart[] = r.content
+    if (r.content_multimodal) {
+      try { content = JSON.parse(r.content_multimodal) } catch { /* ignore */ }
+    }
+    const msg: LLMMessage = { role: r.role, content }
+    if (r.tool_calls_json) {
+      try { msg.tool_calls = JSON.parse(r.tool_calls_json) } catch { /* ignore */ }
+    }
+    if (r.tool_call_id) msg.tool_call_id = r.tool_call_id
+    if (r.reasoning_content) msg.reasoning_content = r.reasoning_content
+    return msg
+  })
+}
+
+// ─── Summaries ────────────────────────────────────────────────────────────────
+
+export interface Summary {
+  summary: string
+  last_message_id: number
+  messages_covered: number
+}
+
+export function getSummary(threadId: string): Summary | null {
+  const db = getDb()
+  return db.query(
+    "SELECT summary, last_message_id, messages_covered FROM summaries WHERE thread_id = ?"
+  ).get(threadId) as Summary | null
+}
+
+export function saveSummary(
+  threadId: string,
+  summary: string,
+  messagesCovered: number,
+  lastMessageId: number
+): void {
+  const db = getDb()
+  db.query(`
+    INSERT INTO summaries (thread_id, summary, messages_covered, last_message_id)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(thread_id) DO UPDATE SET
+      summary        = excluded.summary,
+      messages_covered = excluded.messages_covered,
+      last_message_id  = excluded.last_message_id,
+      updated_at       = unixepoch()
+  `).run(threadId, summary, messagesCovered, lastMessageId)
+}
+
+// ─── Scratchpad ───────────────────────────────────────────────────────────────
+
+export function saveScratchpadNote(
+  threadId: string,
+  key: string,
+  value: string,
+  source?: string
+): void {
+  const db = getDb()
+  db.query(`
+    INSERT INTO scratchpad (thread_id, key, value, source)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(thread_id, key) DO UPDATE SET
+      value      = excluded.value,
+      source     = excluded.source,
+      updated_at = unixepoch()
+  `).run(threadId, key, value, source ?? null)
+}
+
+export function getScratchpad(threadId: string): Array<{ key: string; value: string }> {
+  const db = getDb()
+  return db.query(
+    "SELECT key, value FROM scratchpad WHERE thread_id = ? ORDER BY updated_at DESC"
+  ).all(threadId) as Array<{ key: string; value: string }>
+}
+
+export function deleteScratchpadNote(threadId: string, key: string): void {
+  const db = getDb()
+  db.query("DELETE FROM scratchpad WHERE thread_id = ? AND key = ?").run(threadId, key)
+}

package/packages/core/src/agent/curator.ts

@@ -0,0 +1,158 @@
+/**
+ * ACE Curator — converts reflections into playbook rules.
+ *
+ * Runs after the Reflector. Performs incremental edits to the playbook:
+ *   - New insights → new rules
+ *   - Repeated patterns → increment helpful_count
+ *   - Contradicted rules → increment harmful_count or deactivate
+ *   - Deactivate rules where harmful_count > helpful_count
+ *   - Archive unused workers
+ *
+ * Never rewrites the whole playbook — only incremental edits.
+ */
+
+import { logger } from "../utils/logger"
+
+const log = logger.child("curator")
+
+const DAYS_BEFORE_ARCHIVE = 14   // archive workers not used in N days
+const MAX_HARMFUL_BEFORE_PRUNE = 3
+
+/** Entry point — called by reflector.ts after it inserts new reflections */
+export async function runCurator(): Promise<void> {
+  try {
+    const { getDb } = await import("../storage/sqlite")
+    const db = getDb()
+
+    // Process unprocessed reflections (those newer than last run)
+    const lastProcessed = (db.query<any, []>(
+      "SELECT COALESCE(MAX(source_reflection_id), 0) as mid FROM playbook"
+    ).get() as any)?.mid ?? 0
+
+    const reflections = (db.query as any)(
+      "SELECT * FROM reflections WHERE id > ? ORDER BY id ASC"
+    ).all(lastProcessed)
+
+    if (reflections.length === 0) {
+      log.debug("[curator] No new reflections to process")
+    } else {
+      log.info(`[curator] Processing ${reflections.length} new reflections`)
+      for (const reflection of reflections) {
+        processReflection(db, reflection)
+      }
+    }
+
+    // Prune rules where harmful > helpful (consistently bad rules)
+    db.query(`
+      UPDATE playbook
+      SET active = 0, updated_at = unixepoch()
+      WHERE active = 1
+        AND harmful_count > helpful_count
+        AND harmful_count >= ?
+    `).run(MAX_HARMFUL_BEFORE_PRUNE)
+
+    // Archive unused workers
+    const cutoff = Math.floor(Date.now() / 1000) - (DAYS_BEFORE_ARCHIVE * 86400)
+    const staleworkers = (db.query as any)(`
+      SELECT a.id, a.name
+      FROM agents a
+      WHERE a.role = 'worker'
+        AND a.status != 'archived'
+        AND a.enabled = 1
+        AND (
+          SELECT MAX(t.created_at) FROM traces t WHERE t.agent_id = a.id
+        ) < ?
+    `).all(cutoff)
+
+    for (const worker of staleworkers) {
+      db.query(
+        "UPDATE agents SET status = 'archived', updated_at = unixepoch() WHERE id = ?"
+      ).run(worker.id)
+
+      // Add playbook note about archival
+      addOrUpdateRule(db, {
+        rule: `Worker '${worker.name}' was archived due to inactivity (>${DAYS_BEFORE_ARCHIVE} days unused).`,
+        category: "agent_creation",
+        applicable_to: null,
+        sourceReflectionId: null,
+      })
+
+      log.info(`[curator] Archived inactive worker: ${worker.name} (${worker.id})`)
+    }
+
+    log.info("[curator] Playbook updated")
+  } catch (err) {
+    log.warn("[curator] Error:", err)
+  }
+}
+
+// ─── Process a single reflection ─────────────────────────────────────────────
+
+function processReflection(db: any, reflection: any): void {
+  const category = mapInsightTypeToCategory(reflection.insight_type)
+
+  // Check if a similar rule already exists (fuzzy check by first 60 chars)
+  const prefix = reflection.description.substring(0, 60)
+  const existing = (db.query as any)(
+    "SELECT id, helpful_count FROM playbook WHERE rule LIKE ? AND active = 1 LIMIT 1"
+  ).get(`${prefix}%`)
+
+  if (existing) {
+    // Reinforce existing rule
+    db.query(
+      "UPDATE playbook SET helpful_count = helpful_count + 1, updated_at = unixepoch() WHERE id = ?"
+    ).run(existing.id)
+    return
+  }
+
+  // Insert new rule
+  db.query(`
+    INSERT INTO playbook (rule, category, applicable_to, helpful_count, source_reflection_id)
+    VALUES (?, ?, ?, 1, ?)
+  `).run(
+    reflection.description,
+    category,
+    reflection.affected_tools
+      ? JSON.stringify(JSON.parse(reflection.affected_tools))
+      : null,
+    reflection.id,
+  )
+}
+
+function mapInsightTypeToCategory(
+  type: string
+): "tool_selection" | "response_quality" | "error_avoidance" | "optimization" | "agent_creation" {
+  const map: Record<string, any> = {
+    success_pattern: "tool_selection",
+    failure_pattern: "error_avoidance",
+    optimization: "optimization",
+    ethics_violation: "error_avoidance",
+  }
+  return map[type] ?? "optimization"
+}
+
+function addOrUpdateRule(
+  db: any,
+  opts: {
+    rule: string
+    category: string
+    applicable_to: string | null
+    sourceReflectionId: number | null
+  }
+): void {
+  const prefix = opts.rule.substring(0, 60)
+  const existing = (db.query as any)(
+    "SELECT id FROM playbook WHERE rule LIKE ? LIMIT 1"
+  ).get(`${prefix}%`)
+
+  if (existing) {
+    db.query(
+      "UPDATE playbook SET helpful_count = helpful_count + 1, updated_at = unixepoch() WHERE id = ?"
+    ).run(existing.id)
+  } else {
+    db.query(`
+      INSERT INTO playbook (rule, category, applicable_to, helpful_count, source_reflection_id)
+      VALUES (?, ?, ?, 1, ?)
+    `).run(opts.rule, opts.category, opts.applicable_to, opts.sourceReflectionId)
+  }
+}

package/packages/core/src/agent/hooks.ts

@@ -0,0 +1,166 @@
+import type { Config } from "../config/loader.ts";
+import { logger } from "../utils/logger.ts";
+import * as childProcess from "node:child_process";
+
+export type HookName =
+  | "before_model_resolve"
+  | "before_prompt_build"
+  | "before_tool_call"
+  | "after_tool_call"
+  | "tool_result_persist"
+  | "before_compaction"
+  | "after_compaction"
+  | "message_received"
+  | "message_sending"
+  | "message_sent"
+  | "session_start"
+  | "session_end"
+  | "gateway_start"
+  | "gateway_stop";
+
+export interface HookContext {
+  sessionId?: string;
+  agentId?: string;
+  data?: Record<string, unknown>;
+  timestamp: Date;
+}
+
+export type HookHandler = (context: HookContext) => Promise<Record<string, unknown> | void>;
+
+export class HookPipeline {
+  private config: Config;
+  private log = logger.child("hooks");
+  private handlers: Map<HookName, HookHandler[]> = new Map();
+  private scriptCache: Map<HookName, string> = new Map();
+
+  constructor(config: Config) {
+    this.config = config;
+    this.loadScripts();
+  }
+
+  private loadScripts(): void {
+    const scripts = this.config.hooks?.scripts;
+    if (!scripts) return;
+
+    const hookNames: HookName[] = [
+      "before_model_resolve",
+      "before_prompt_build",
+      "before_tool_call",
+      "after_tool_call",
+      "tool_result_persist",
+      "before_compaction",
+      "after_compaction",
+      "message_received",
+      "message_sending",
+      "message_sent",
+      "session_start",
+      "session_end",
+      "gateway_start",
+      "gateway_stop",
+    ];
+
+    for (const name of hookNames) {
+      const script = scripts[name];
+      if (script) {
+        this.scriptCache.set(name, script);
+        this.log.debug(`Loaded script for hook: ${name}`);
+      }
+    }
+  }
+
+  registerHandler(name: HookName, handler: HookHandler): void {
+    const handlers = this.handlers.get(name) ?? [];
+    handlers.push(handler);
+    this.handlers.set(name, handlers);
+    this.log.debug(`Registered handler for hook: ${name}`);
+  }
+
+  unregisterHandler(name: HookName, handler: HookHandler): boolean {
+    const handlers = this.handlers.get(name);
+    if (!handlers) return false;
+
+    const index = handlers.indexOf(handler);
+    if (index >= 0) {
+      handlers.splice(index, 1);
+      return true;
+    }
+    return false;
+  }
+
+  async execute(name: HookName, context: HookContext): Promise<Record<string, unknown> | void> {
+    this.log.debug(`Executing hook: ${name}`, { sessionId: context.sessionId });
+
+    const handlers = this.handlers.get(name) ?? [];
+    for (const handler of handlers) {
+      try {
+        await handler(context);
+      } catch (error) {
+        this.log.error(`Handler failed for ${name}: ${(error as Error).message}`);
+      }
+    }
+
+    const script = this.scriptCache.get(name);
+    if (script) {
+      try {
+        const result = await this.executeScript(script, context);
+        return result;
+      } catch (error) {
+        this.log.error(`Script failed for ${name}: ${(error as Error).message}`);
+      }
+    }
+  }
+
+  private async executeScript(
+    scriptPath: string,
+    context: HookContext
+  ): Promise<Record<string, unknown> | void> {
+    return new Promise((resolve, reject) => {
+      const payload = JSON.stringify(context);
+
+      const proc = childProcess.spawn(scriptPath, [], {
+        stdio: ["pipe", "pipe", "pipe"],
+        shell: true,
+      }) as any;
+
+      let stdout = "";
+      let stderr = "";
+
+      proc.stdout?.on("data", (data) => {
+        stdout += data.toString();
+      });
+
+      proc.stderr?.on("data", (data) => {
+        stderr += data.toString();
+      });
+
+      proc.on("close", (code) => {
+        if (code === 0 && stdout) {
+          try {
+            resolve(JSON.parse(stdout));
+          } catch {
+            resolve();
+          }
+        } else if (stderr) {
+          reject(new Error(stderr));
+        } else {
+          resolve();
+        }
+      });
+
+      proc.on("error", (error) => {
+        reject(error);
+      });
+
+      proc.stdin?.write(payload);
+      proc.stdin?.end();
+    });
+  }
+
+  hasHandlers(name: HookName): boolean {
+    return (this.handlers.get(name)?.length ?? 0) > 0 || this.scriptCache.has(name);
+  }
+}
+
+export function createHookPipeline(config: Config): HookPipeline {
+  return new HookPipeline(config);
+}