@johpaz/hive 1.1.0

package/packages/core/src/agent/conversation-store.ts

@@ -0,0 +1,198 @@
+/**
+ * Conversation Store — persists message history in the `conversations` table.
+ * Replaces the LangGraph BunSqliteSaver + lg_checkpoints approach.
+ *
+ * Also manages: summaries, scratchpad.
+ */
+
+import { getDb } from "../storage/sqlite"
+import { logger } from "../utils/logger"
+import type { LLMMessage } from "./llm-client"
+
+const log = logger.child("conv-store")
+
+// ─── Types ────────────────────────────────────────────────────────────────────
+
+export interface StoredMessage {
+  id: number
+  thread_id: string
+  channel: string
+  role: "user" | "assistant" | "tool" | "system"
+  content: string
+  tool_calls_json: string | null
+  tool_call_id: string | null
+  token_count: number
+  created_at: number
+}
+
+// ─── Token estimation ─────────────────────────────────────────────────────────
+
+export function estimateTokens(text: string): number {
+  return Math.ceil((text ?? "").length / 4)
+}
+
+// ─── Message operations ───────────────────────────────────────────────────────
+
+export function addMessage(
+  threadId: string,
+  role: StoredMessage["role"],
+  content: string,
+  opts?: {
+    channel?: string
+    tool_calls?: LLMMessage["tool_calls"]
+    tool_call_id?: string
+  }
+): number {
+  const db = getDb()
+  const tool_calls_json = opts?.tool_calls?.length
+    ? JSON.stringify(opts.tool_calls)
+    : null
+
+  const result = db.query(`
+    INSERT INTO conversations (thread_id, channel, role, content, tool_calls_json, tool_call_id, token_count)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+    RETURNING id
+  `).get(
+    threadId,
+    opts?.channel ?? "webchat",
+    role,
+    content,
+    tool_calls_json,
+    opts?.tool_call_id ?? null,
+    estimateTokens(content) + estimateTokens(tool_calls_json ?? ""),
+  ) as { id: number }
+
+  return result.id
+}
+
+/**
+ * Returns all messages for the thread ordered oldest → newest.
+ */
+export function getHistory(threadId: string, limit = 200): StoredMessage[] {
+  const db = getDb()
+  return db.query(`
+    SELECT * FROM conversations
+    WHERE thread_id = ?
+    ORDER BY id ASC
+    LIMIT ?
+  `).all(threadId, limit) as StoredMessage[]
+}
+
+/**
+ * Returns only the last N messages (oldest → newest order).
+ */
+export function getRecentMessages(threadId: string, n: number): StoredMessage[] {
+  const db = getDb()
+  const rows = db.query(`
+    SELECT * FROM conversations
+    WHERE thread_id = ?
+    ORDER BY id DESC
+    LIMIT ?
+  `).all(threadId, n) as StoredMessage[]
+  return rows.reverse()
+}
+
+export function getMessageCount(threadId: string): number {
+  const db = getDb()
+  const row = db.query(
+    "SELECT COUNT(*) as cnt FROM conversations WHERE thread_id = ?"
+  ).get(threadId) as { cnt: number }
+  return row.cnt
+}
+
+export function getTotalTokens(threadId: string): number {
+  const db = getDb()
+  const row = db.query(
+    "SELECT COALESCE(SUM(token_count), 0) as total FROM conversations WHERE thread_id = ?"
+  ).get(threadId) as { total: number }
+  return row.total
+}
+
+/**
+ * Messages after a given message ID (for incremental summary updates).
+ */
+export function getMessagesAfter(threadId: string, afterId: number): StoredMessage[] {
+  const db = getDb()
+  return db.query(`
+    SELECT * FROM conversations
+    WHERE thread_id = ? AND id > ?
+    ORDER BY id ASC
+  `).all(threadId, afterId) as StoredMessage[]
+}
+
+// ─── Convert stored messages → LLMMessage array ───────────────────────────────
+
+export function toAPIMessages(rows: StoredMessage[]): LLMMessage[] {
+  return rows.map((r) => {
+    const msg: LLMMessage = { role: r.role, content: r.content }
+    if (r.tool_calls_json) {
+      try { msg.tool_calls = JSON.parse(r.tool_calls_json) } catch { /* ignore */ }
+    }
+    if (r.tool_call_id) msg.tool_call_id = r.tool_call_id
+    return msg
+  })
+}
+
+// ─── Summaries ────────────────────────────────────────────────────────────────
+
+export interface Summary {
+  summary: string
+  last_message_id: number
+  messages_covered: number
+}
+
+export function getSummary(threadId: string): Summary | null {
+  const db = getDb()
+  return db.query(
+    "SELECT summary, last_message_id, messages_covered FROM summaries WHERE thread_id = ?"
+  ).get(threadId) as Summary | null
+}
+
+export function saveSummary(
+  threadId: string,
+  summary: string,
+  messagesCovered: number,
+  lastMessageId: number
+): void {
+  const db = getDb()
+  db.query(`
+    INSERT INTO summaries (thread_id, summary, messages_covered, last_message_id)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(thread_id) DO UPDATE SET
+      summary        = excluded.summary,
+      messages_covered = excluded.messages_covered,
+      last_message_id  = excluded.last_message_id,
+      updated_at       = unixepoch()
+  `).run(threadId, summary, messagesCovered, lastMessageId)
+}
+
+// ─── Scratchpad ───────────────────────────────────────────────────────────────
+
+export function saveScratchpadNote(
+  threadId: string,
+  key: string,
+  value: string,
+  source?: string
+): void {
+  const db = getDb()
+  db.query(`
+    INSERT INTO scratchpad (thread_id, key, value, source)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(thread_id, key) DO UPDATE SET
+      value      = excluded.value,
+      source     = excluded.source,
+      updated_at = unixepoch()
+  `).run(threadId, key, value, source ?? null)
+}
+
+export function getScratchpad(threadId: string): Array<{ key: string; value: string }> {
+  const db = getDb()
+  return db.query(
+    "SELECT key, value FROM scratchpad WHERE thread_id = ? ORDER BY updated_at DESC"
+  ).all(threadId) as Array<{ key: string; value: string }>
+}
+
+export function deleteScratchpadNote(threadId: string, key: string): void {
+  const db = getDb()
+  db.query("DELETE FROM scratchpad WHERE thread_id = ? AND key = ?").run(threadId, key)
+}

package/packages/core/src/agent/curator.ts

@@ -0,0 +1,158 @@
+/**
+ * ACE Curator — converts reflections into playbook rules.
+ *
+ * Runs after the Reflector. Performs incremental edits to the playbook:
+ *   - New insights → new rules
+ *   - Repeated patterns → increment helpful_count
+ *   - Contradicted rules → increment harmful_count or deactivate
+ *   - Deactivate rules where harmful_count > helpful_count
+ *   - Archive unused workers
+ *
+ * Never rewrites the whole playbook — only incremental edits.
+ */
+
+import { logger } from "../utils/logger"
+
+const log = logger.child("curator")
+
+const DAYS_BEFORE_ARCHIVE = 14   // archive workers not used in N days
+const MAX_HARMFUL_BEFORE_PRUNE = 3
+
+/** Entry point — called by reflector.ts after it inserts new reflections */
+export async function runCurator(): Promise<void> {
+  try {
+    const { getDb } = await import("../storage/sqlite")
+    const db = getDb()
+
+    // Process unprocessed reflections (those newer than last run)
+    const lastProcessed = (db.query<any, []>(
+      "SELECT COALESCE(MAX(source_reflection_id), 0) as mid FROM playbook"
+    ).get() as any)?.mid ?? 0
+
+    const reflections = (db.query as any)(
+      "SELECT * FROM reflections WHERE id > ? ORDER BY id ASC"
+    ).all(lastProcessed)
+
+    if (reflections.length === 0) {
+      log.debug("[curator] No new reflections to process")
+    } else {
+      log.info(`[curator] Processing ${reflections.length} new reflections`)
+      for (const reflection of reflections) {
+        processReflection(db, reflection)
+      }
+    }
+
+    // Prune rules where harmful > helpful (consistently bad rules)
+    db.query(`
+      UPDATE playbook
+      SET active = 0, updated_at = unixepoch()
+      WHERE active = 1
+        AND harmful_count > helpful_count
+        AND harmful_count >= ?
+    `).run(MAX_HARMFUL_BEFORE_PRUNE)
+
+    // Archive unused workers
+    const cutoff = Math.floor(Date.now() / 1000) - (DAYS_BEFORE_ARCHIVE * 86400)
+    const staleworkers = (db.query as any)(`
+      SELECT a.id, a.name
+      FROM agents a
+      WHERE a.role = 'worker'
+        AND a.status != 'archived'
+        AND a.enabled = 1
+        AND (
+          SELECT MAX(t.created_at) FROM traces t WHERE t.agent_id = a.id
+        ) < ?
+    `).all(cutoff)
+
+    for (const worker of staleworkers) {
+      db.query(
+        "UPDATE agents SET status = 'archived', updated_at = unixepoch() WHERE id = ?"
+      ).run(worker.id)
+
+      // Add playbook note about archival
+      addOrUpdateRule(db, {
+        rule: `Worker '${worker.name}' was archived due to inactivity (>${DAYS_BEFORE_ARCHIVE} days unused).`,
+        category: "agent_creation",
+        applicable_to: null,
+        sourceReflectionId: null,
+      })
+
+      log.info(`[curator] Archived inactive worker: ${worker.name} (${worker.id})`)
+    }
+
+    log.info("[curator] Playbook updated")
+  } catch (err) {
+    log.warn("[curator] Error:", err)
+  }
+}
+
+// ─── Process a single reflection ─────────────────────────────────────────────
+
+function processReflection(db: any, reflection: any): void {
+  const category = mapInsightTypeToCategory(reflection.insight_type)
+
+  // Check if a similar rule already exists (fuzzy check by first 60 chars)
+  const prefix = reflection.description.substring(0, 60)
+  const existing = (db.query as any)(
+    "SELECT id, helpful_count FROM playbook WHERE rule LIKE ? AND active = 1 LIMIT 1"
+  ).get(`${prefix}%`)
+
+  if (existing) {
+    // Reinforce existing rule
+    db.query(
+      "UPDATE playbook SET helpful_count = helpful_count + 1, updated_at = unixepoch() WHERE id = ?"
+    ).run(existing.id)
+    return
+  }
+
+  // Insert new rule
+  db.query(`
+    INSERT INTO playbook (rule, category, applicable_to, helpful_count, source_reflection_id)
+    VALUES (?, ?, ?, 1, ?)
+  `).run(
+    reflection.description,
+    category,
+    reflection.affected_tools
+      ? JSON.stringify(JSON.parse(reflection.affected_tools))
+      : null,
+    reflection.id,
+  )
+}
+
+function mapInsightTypeToCategory(
+  type: string
+): "tool_selection" | "response_quality" | "error_avoidance" | "optimization" | "agent_creation" {
+  const map: Record<string, any> = {
+    success_pattern: "tool_selection",
+    failure_pattern: "error_avoidance",
+    optimization: "optimization",
+    ethics_violation: "error_avoidance",
+  }
+  return map[type] ?? "optimization"
+}
+
+function addOrUpdateRule(
+  db: any,
+  opts: {
+    rule: string
+    category: string
+    applicable_to: string | null
+    sourceReflectionId: number | null
+  }
+): void {
+  const prefix = opts.rule.substring(0, 60)
+  const existing = (db.query as any)(
+    "SELECT id FROM playbook WHERE rule LIKE ? LIMIT 1"
+  ).get(`${prefix}%`)
+
+  if (existing) {
+    db.query(
+      "UPDATE playbook SET helpful_count = helpful_count + 1, updated_at = unixepoch() WHERE id = ?"
+    ).run(existing.id)
+  } else {
+    db.query(`
+      INSERT INTO playbook (rule, category, applicable_to, helpful_count, source_reflection_id)
+      VALUES (?, ?, ?, 1, ?)
+    `).run(opts.rule, opts.category, opts.applicable_to, opts.sourceReflectionId)
+  }
+}

package/packages/core/src/agent/hooks.ts

@@ -0,0 +1,166 @@
+import type { Config } from "../config/loader.ts";
+import { logger } from "../utils/logger.ts";
+import * as childProcess from "node:child_process";
+
+export type HookName =
+  | "before_model_resolve"
+  | "before_prompt_build"
+  | "before_tool_call"
+  | "after_tool_call"
+  | "tool_result_persist"
+  | "before_compaction"
+  | "after_compaction"
+  | "message_received"
+  | "message_sending"
+  | "message_sent"
+  | "session_start"
+  | "session_end"
+  | "gateway_start"
+  | "gateway_stop";
+
+export interface HookContext {
+  sessionId?: string;
+  agentId?: string;
+  data?: Record<string, unknown>;
+  timestamp: Date;
+}
+
+export type HookHandler = (context: HookContext) => Promise<Record<string, unknown> | void>;
+
+export class HookPipeline {
+  private config: Config;
+  private log = logger.child("hooks");
+  private handlers: Map<HookName, HookHandler[]> = new Map();
+  private scriptCache: Map<HookName, string> = new Map();
+
+  constructor(config: Config) {
+    this.config = config;
+    this.loadScripts();
+  }
+
+  private loadScripts(): void {
+    const scripts = this.config.hooks?.scripts;
+    if (!scripts) return;
+
+    const hookNames: HookName[] = [
+      "before_model_resolve",
+      "before_prompt_build",
+      "before_tool_call",
+      "after_tool_call",
+      "tool_result_persist",
+      "before_compaction",
+      "after_compaction",
+      "message_received",
+      "message_sending",
+      "message_sent",
+      "session_start",
+      "session_end",
+      "gateway_start",
+      "gateway_stop",
+    ];
+
+    for (const name of hookNames) {
+      const script = scripts[name];
+      if (script) {
+        this.scriptCache.set(name, script);
+        this.log.debug(`Loaded script for hook: ${name}`);
+      }
+    }
+  }
+
+  registerHandler(name: HookName, handler: HookHandler): void {
+    const handlers = this.handlers.get(name) ?? [];
+    handlers.push(handler);
+    this.handlers.set(name, handlers);
+    this.log.debug(`Registered handler for hook: ${name}`);
+  }
+
+  unregisterHandler(name: HookName, handler: HookHandler): boolean {
+    const handlers = this.handlers.get(name);
+    if (!handlers) return false;
+
+    const index = handlers.indexOf(handler);
+    if (index >= 0) {
+      handlers.splice(index, 1);
+      return true;
+    }
+    return false;
+  }
+
+  async execute(name: HookName, context: HookContext): Promise<Record<string, unknown> | void> {
+    this.log.debug(`Executing hook: ${name}`, { sessionId: context.sessionId });
+
+    const handlers = this.handlers.get(name) ?? [];
+    for (const handler of handlers) {
+      try {
+        await handler(context);
+      } catch (error) {
+        this.log.error(`Handler failed for ${name}: ${(error as Error).message}`);
+      }
+    }
+
+    const script = this.scriptCache.get(name);
+    if (script) {
+      try {
+        const result = await this.executeScript(script, context);
+        return result;
+      } catch (error) {
+        this.log.error(`Script failed for ${name}: ${(error as Error).message}`);
+      }
+    }
+  }
+
+  private async executeScript(
+    scriptPath: string,
+    context: HookContext
+  ): Promise<Record<string, unknown> | void> {
+    return new Promise((resolve, reject) => {
+      const payload = JSON.stringify(context);
+
+      const proc = childProcess.spawn(scriptPath, [], {
+        stdio: ["pipe", "pipe", "pipe"],
+        shell: true,
+      }) as any;
+
+      let stdout = "";
+      let stderr = "";
+
+      proc.stdout?.on("data", (data) => {
+        stdout += data.toString();
+      });
+
+      proc.stderr?.on("data", (data) => {
+        stderr += data.toString();
+      });
+
+      proc.on("close", (code) => {
+        if (code === 0 && stdout) {
+          try {
+            resolve(JSON.parse(stdout));
+          } catch {
+            resolve();
+          }
+        } else if (stderr) {
+          reject(new Error(stderr));
+        } else {
+          resolve();
+        }
+      });
+
+      proc.on("error", (error) => {
+        reject(error);
+      });
+
+      proc.stdin?.write(payload);
+      proc.stdin?.end();
+    });
+  }
+
+  hasHandlers(name: HookName): boolean {
+    return (this.handlers.get(name)?.length ?? 0) > 0 || this.scriptCache.has(name);
+  }
+}
+
+export function createHookPipeline(config: Config): HookPipeline {
+  return new HookPipeline(config);
+}

package/packages/core/src/agent/index.ts

@@ -0,0 +1,116 @@
+import type { Config } from "../config/loader.ts"
+import { logger } from "../utils/logger.ts"
+import { MCPClientManager, createMCPManager } from "@johpaz/hive-mcp"
+import { getDb } from "../storage/sqlite.ts"
+import { buildSystemPromptWithProjects } from "./prompt-builder.ts"
+import { EventEmitter } from "node:events"
+
+export interface AgentOptions {
+  agentId: string
+  config: Config
+  workspacePath: string
+}
+
+/**
+ * Agent — minimal wrapper kept only to manage the MCP connection lifecycle.
+ * Tool execution is handled entirely by native-tools.ts + agent-loop.ts.
+ */
+export class Agent extends EventEmitter {
+  readonly agentId: string
+  readonly workspacePath: string
+  private config: Config
+  private mcpManager: MCPClientManager | null = null
+  private log = logger.child("agent")
+
+  constructor(options: AgentOptions) {
+    super()
+    this.agentId = options.agentId
+    this.workspacePath = options.workspacePath
+    this.config = options.config
+  }
+
+  async initialize(): Promise<void> {
+    this.log.info(`Initializing agent: ${this.agentId}`)
+
+    if (this.config.mcp?.enabled !== false) {
+      const db = getDb()
+      const dbServers = db.query<any, []>("SELECT * FROM mcp_servers").all()
+      const mcpConfig = { ...this.config.mcp }
+      mcpConfig.servers = { ...(mcpConfig.servers || {}) }
+
+      for (const s of dbServers) {
+        if (s.enabled) {
+          let headers: Record<string, string> | undefined
+          if (s.headers_encrypted && s.headers_iv) {
+            try {
+              const { decryptConfig } = await import("../storage/crypto.ts")
+              headers = decryptConfig(s.headers_encrypted, s.headers_iv)
+            } catch (e) {
+              this.log.error(`Failed to decrypt headers for ${s.name}: ${(e as Error).message}`)
+            }
+          }
+          mcpConfig.servers[s.name] = {
+            transport: s.transport,
+            command: s.command,
+            args: s.args ? JSON.parse(s.args) : [],
+            url: s.url,
+            headers,
+            enabled: true,
+            ...mcpConfig.servers[s.name],
+          }
+          if (headers) mcpConfig.servers[s.name].headers = headers
+        }
+      }
+
+      this.mcpManager = createMCPManager(mcpConfig)
+
+      const mcpLog = this.log.child("mcp")
+      this.mcpManager.setLogHandler((level, context, message, meta) => {
+        const ctx = context ? `[${context}] ` : ""
+        switch (level) {
+          case "debug": mcpLog.debug(`${ctx}${message}`, meta); break
+          case "info":  mcpLog.info(`${ctx}${message}`, meta);  break
+          case "warn":  mcpLog.warn(`${ctx}${message}`, meta);  break
+          case "error": mcpLog.error(`${ctx}${message}`, meta); break
+        }
+      })
+
+      await this.mcpManager.initialize()
+      try {
+        await this.mcpManager.connectAll()
+      } catch (error) {
+        this.log.warn(`MCP connection errors: ${(error as Error).message}`)
+      }
+    }
+
+    this.log.info(`Agent initialized: ${this.agentId}`)
+  }
+
+  async updateConfig(config: Config): Promise<void> {
+    this.config = config
+    if (this.mcpManager) {
+      await this.mcpManager.updateConfig(this.config.mcp || {})
+    }
+  }
+
+  getConfig(): Config { return this.config }
+  getWorkspacePath(): string { return this.workspacePath }
+  getMCPManager(): MCPClientManager | null { return this.mcpManager }
+
+  // ── Compatibility stubs ──────────────────────────────────────────────────────
+  // The native loop (context-compiler.ts) reads all data from SQLite on every
+  // call, so explicit reload methods are no-ops. buildPrompt() delegates to the
+  // shared prompt builder used by the loop.
+
+  async buildPrompt(): Promise<string> {
+    const userId = process.env.HIVE_USER_ID || ""
+    return buildSystemPromptWithProjects({ agentId: this.agentId, userId })
+  }
+
+  setEthics(_content: string): void { /* ethics loaded from DB by context-compiler */ }
+  async reload(): Promise<void> { /* no-op: context-compiler reads DB on every call */ }
+  async reloadSoul(): Promise<void> { /* no-op */ }
+  async reloadUser(): Promise<void> { /* no-op */ }
+  async reloadEthics(): Promise<void> { /* no-op */ }
+  reloadSkills(): void { /* no-op */ }
+}