@yeaft/webchat-agent 0.1.408 → 0.1.410

package/unify/mcp.js

@@ -0,0 +1,433 @@
+/**
+ * mcp.js — MCP (Model Context Protocol) client manager
+ *
+ * Manages connections to MCP servers and provides a bridge
+ * for the mcp_list_tools and mcp_call_tool tools.
+ *
+ * MCP servers are configured in ~/.yeaft/config.md:
+ *   ---
+ *   mcp_servers:
+ *     - name: github
+ *       command: npx @mcp/github
+ *       args: []
+ *       env:
+ *         GITHUB_TOKEN: ghp_...
+ *     - name: slack
+ *       command: npx @mcp/slack
+ *       args: []
+ *   ---
+ *
+ * Each MCP server communicates via stdio JSON-RPC (MCP protocol).
+ *
+ * Reference: yeaft-unify-design.md §8, yeaft-unify-core-systems.md §3.2
+ */
+
+import { spawn } from 'child_process';
+import { randomUUID } from 'crypto';
+import { EventEmitter } from 'events';
+
+// ─── Constants ──────────────────────────────────────────────
+
+/** Default tool call timeout (30 seconds). */
+const DEFAULT_TIMEOUT_MS = 30000;
+
+/** Server startup timeout (10 seconds). */
+const STARTUP_TIMEOUT_MS = 10000;
+
+// ─── MCP Server Connection ─────────────────────────────────
+
+/**
+ * A single MCP server connection.
+ * Communicates via stdio JSON-RPC.
+ */
+class MCPServerConnection extends EventEmitter {
+  #name;
+  #process;
+  #pendingRequests;
+  #tools;
+  #ready;
+  #buffer;
+
+  /**
+   * @param {string} name — server name (e.g. "github")
+   * @param {{ command: string, args?: string[], env?: object }} config
+   */
+  constructor(name, config) {
+    super();
+    this.#name = name;
+    this.#process = null;
+    this.#pendingRequests = new Map();
+    this.#tools = [];
+    this.#ready = false;
+    this.#buffer = '';
+    this.config = config;
+  }
+
+  get name() { return this.#name; }
+  get tools() { return this.#tools; }
+  get ready() { return this.#ready; }
+
+  /**
+   * Start the MCP server process and initialize.
+   * @returns {Promise<void>}
+   */
+  async start() {
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        reject(new Error(`MCP server "${this.#name}" startup timeout (${STARTUP_TIMEOUT_MS}ms)`));
+      }, STARTUP_TIMEOUT_MS);
+
+      try {
+        this.#process = spawn(this.config.command, this.config.args || [], {
+          stdio: ['pipe', 'pipe', 'pipe'],
+          env: { ...process.env, ...this.config.env },
+        });
+
+        this.#process.stdout.on('data', (data) => {
+          this.#buffer += data.toString();
+          this.#processBuffer();
+        });
+
+        this.#process.stderr.on('data', (data) => {
+          this.emit('error', new Error(`[${this.#name}] stderr: ${data.toString().trim()}`));
+        });
+
+        this.#process.on('close', (code) => {
+          this.#ready = false;
+          this.emit('close', code);
+          // Reject all pending requests
+          for (const [, { reject: rej }] of this.#pendingRequests) {
+            rej(new Error(`MCP server "${this.#name}" closed with code ${code}`));
+          }
+          this.#pendingRequests.clear();
+        });
+
+        this.#process.on('error', (err) => {
+          clearTimeout(timer);
+          reject(err);
+        });
+
+        // Initialize MCP protocol
+        this.#initialize().then(() => {
+          clearTimeout(timer);
+          this.#ready = true;
+          resolve();
+        }).catch((err) => {
+          clearTimeout(timer);
+          reject(err);
+        });
+
+      } catch (err) {
+        clearTimeout(timer);
+        reject(err);
+      }
+    });
+  }
+
+  /**
+   * Send the MCP initialize request and list tools.
+   */
+  async #initialize() {
+    // Send initialize
+    await this.#rpcCall('initialize', {
+      protocolVersion: '2024-11-05',
+      capabilities: {},
+      clientInfo: { name: 'yeaft', version: '0.1.0' },
+    });
+
+    // Send initialized notification
+    this.#rpcNotify('notifications/initialized', {});
+
+    // List tools
+    const result = await this.#rpcCall('tools/list', {});
+    this.#tools = (result?.tools || []).map(tool => ({
+      name: `${this.#name}__${tool.name}`,
+      serverName: this.#name,
+      originalName: tool.name,
+      description: tool.description || '',
+      inputSchema: tool.inputSchema || { type: 'object', properties: {} },
+    }));
+  }
+
+  /**
+   * Call a tool on this MCP server.
+   *
+   * @param {string} toolName — original tool name (without server prefix)
+   * @param {object} args — tool arguments
+   * @param {number} [timeoutMs] — timeout in milliseconds
+   * @returns {Promise<object>}
+   */
+  async callTool(toolName, args = {}, timeoutMs = DEFAULT_TIMEOUT_MS) {
+    const result = await this.#rpcCall('tools/call', {
+      name: toolName,
+      arguments: args,
+    }, timeoutMs);
+
+    return result;
+  }
+
+  /**
+   * Send a JSON-RPC request and wait for response.
+   */
+  async #rpcCall(method, params, timeoutMs = DEFAULT_TIMEOUT_MS) {
+    const id = randomUUID();
+
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.#pendingRequests.delete(id);
+        reject(new Error(`RPC call "${method}" to "${this.#name}" timed out (${timeoutMs}ms)`));
+      }, timeoutMs);
+
+      this.#pendingRequests.set(id, {
+        resolve: (result) => {
+          clearTimeout(timer);
+          this.#pendingRequests.delete(id);
+          resolve(result);
+        },
+        reject: (err) => {
+          clearTimeout(timer);
+          this.#pendingRequests.delete(id);
+          reject(err);
+        },
+      });
+
+      const message = JSON.stringify({ jsonrpc: '2.0', id, method, params });
+      this.#process.stdin.write(message + '\n');
+    });
+  }
+
+  /**
+   * Send a JSON-RPC notification (no response expected).
+   */
+  #rpcNotify(method, params) {
+    const message = JSON.stringify({ jsonrpc: '2.0', method, params });
+    this.#process.stdin.write(message + '\n');
+  }
+
+  /**
+   * Process the stdout buffer for complete JSON-RPC messages.
+   */
+  #processBuffer() {
+    const lines = this.#buffer.split('\n');
+    this.#buffer = lines.pop() || ''; // Keep incomplete line in buffer
+
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+
+      try {
+        const msg = JSON.parse(trimmed);
+
+        if (msg.id && this.#pendingRequests.has(msg.id)) {
+          const pending = this.#pendingRequests.get(msg.id);
+          if (msg.error) {
+            pending.reject(new Error(`MCP error: ${msg.error.message || JSON.stringify(msg.error)}`));
+          } else {
+            pending.resolve(msg.result);
+          }
+        }
+
+        // Handle notifications from server
+        if (!msg.id && msg.method) {
+          this.emit('notification', msg);
+        }
+      } catch {
+        // Not valid JSON — ignore (could be debug output)
+      }
+    }
+  }
+
+  /**
+   * Stop the MCP server process.
+   */
+  async stop() {
+    if (this.#process) {
+      this.#ready = false;
+      this.#process.kill('SIGTERM');
+      // Wait a bit then force kill
+      await new Promise(resolve => setTimeout(resolve, 2000));
+      if (this.#process && !this.#process.killed) {
+        this.#process.kill('SIGKILL');
+      }
+      this.#process = null;
+    }
+  }
+}
+
+// ─── MCP Manager ────────────────────────────────────────────
+
+/**
+ * MCPManager — manages multiple MCP server connections.
+ */
+export class MCPManager {
+  /** @type {Map<string, MCPServerConnection>} */
+  #servers = new Map();
+
+  /** @type {Map<string, object>} */
+  #toolIndex = new Map(); // fullToolName → { server, tool }
+
+  /**
+   * Connect to an MCP server.
+   *
+   * @param {{ name: string, command: string, args?: string[], env?: object }} serverConfig
+   * @returns {Promise<{ name: string, toolCount: number }>}
+   */
+  async connect(serverConfig) {
+    const { name } = serverConfig;
+
+    // Disconnect existing if any
+    if (this.#servers.has(name)) {
+      await this.disconnect(name);
+    }
+
+    const connection = new MCPServerConnection(name, serverConfig);
+    await connection.start();
+
+    this.#servers.set(name, connection);
+
+    // Index tools
+    for (const tool of connection.tools) {
+      this.#toolIndex.set(tool.name, { server: connection, tool });
+    }
+
+    return { name, toolCount: connection.tools.length };
+  }
+
+  /**
+   * Connect to all servers from config.
+   *
+   * @param {object[]} serverConfigs — array of server configurations
+   * @returns {Promise<{ connected: string[], failed: { name: string, error: string }[] }>}
+   */
+  async connectAll(serverConfigs) {
+    const connected = [];
+    const failed = [];
+
+    for (const config of serverConfigs) {
+      try {
+        await this.connect(config);
+        connected.push(config.name);
+      } catch (err) {
+        failed.push({ name: config.name, error: err.message });
+      }
+    }
+
+    return { connected, failed };
+  }
+
+  /**
+   * Disconnect from a specific MCP server.
+   *
+   * @param {string} name — server name
+   */
+  async disconnect(name) {
+    const connection = this.#servers.get(name);
+    if (connection) {
+      // Remove tools from index
+      for (const tool of connection.tools) {
+        this.#toolIndex.delete(tool.name);
+      }
+      await connection.stop();
+      this.#servers.delete(name);
+    }
+  }
+
+  /**
+   * Disconnect from all servers.
+   */
+  async disconnectAll() {
+    const names = [...this.#servers.keys()];
+    for (const name of names) {
+      await this.disconnect(name);
+    }
+  }
+
+  /**
+   * List all available tools from all connected servers.
+   *
+   * @param {string} [serverFilter] — optionally filter by server name
+   * @returns {object[]}
+   */
+  listTools(serverFilter) {
+    const tools = [];
+
+    for (const [, { server, tool }] of this.#toolIndex) {
+      if (serverFilter && server.name !== serverFilter) continue;
+      tools.push({
+        name: tool.name,
+        server: server.name,
+        description: tool.description,
+        inputSchema: tool.inputSchema,
+      });
+    }
+
+    return tools;
+  }
+
+  /**
+   * Call a tool by its full name (server__toolName).
+   *
+   * @param {string} fullToolName — e.g. "github__list_prs"
+   * @param {object} [args={}] — tool arguments
+   * @param {number} [timeoutMs] — timeout
+   * @returns {Promise<object>}
+   */
+  async callTool(fullToolName, args = {}, timeoutMs = DEFAULT_TIMEOUT_MS) {
+    const entry = this.#toolIndex.get(fullToolName);
+    if (!entry) {
+      throw new Error(`MCP tool "${fullToolName}" not found. Available: ${[...this.#toolIndex.keys()].join(', ') || '(none)'}`);
+    }
+
+    const { server, tool } = entry;
+    if (!server.ready) {
+      throw new Error(`MCP server "${server.name}" is not ready`);
+    }
+
+    return server.callTool(tool.originalName, args, timeoutMs);
+  }
+
+  /**
+   * Get the status of all servers.
+   *
+   * @returns {{ name: string, ready: boolean, toolCount: number }[]}
+   */
+  status() {
+    return [...this.#servers.entries()].map(([name, conn]) => ({
+      name,
+      ready: conn.ready,
+      toolCount: conn.tools.length,
+    }));
+  }
+
+  /**
+   * Check if any servers are connected.
+   * @returns {boolean}
+   */
+  get hasServers() {
+    return this.#servers.size > 0;
+  }
+
+  /**
+   * Total tool count across all servers.
+   * @returns {number}
+   */
+  get toolCount() {
+    return this.#toolIndex.size;
+  }
+}
+
+/**
+ * Create an MCPManager and optionally connect to servers from config.
+ *
+ * @param {object} [config] — { mcp_servers: [...] }
+ * @returns {Promise<MCPManager>}
+ */
+export async function createMCPManager(config) {
+  const manager = new MCPManager();
+
+  if (config?.mcp_servers && Array.isArray(config.mcp_servers)) {
+    await manager.connectAll(config.mcp_servers);
+  }
+
+  return manager;
+}

package/unify/memory/consolidate.js

@@ -0,0 +1,187 @@
+/**
+ * consolidate.js — Consolidate = compact + extract (one LLM call)
+ *
+ * Triggered when hot_tokens > MESSAGE_TOKEN_BUDGET.
+ * One LLM call does two things simultaneously:
+ *   1. Generate compact summary → append to compact.md ("short-term memory")
+ *   2. Extract memory entries → write to entries/ ("long-term memory")
+ *
+ * After consolidation:
+ *   - Processed messages moved from messages/ to cold/
+ *   - index.md + scopes.md updated
+ *
+ * Reference: yeaft-unify-core-systems.md §3.1, §4.2
+ *            yeaft-unify-design.md §6.1
+ */
+
+import { extractMemories } from './extract.js';
+
+// ─── Constants ──────────────────────────────────────────────────
+
+/** Default MESSAGE_TOKEN_BUDGET (context * 4%, default ~8192). */
+export const DEFAULT_MESSAGE_TOKEN_BUDGET = 8192;
+
+/** After compact, keep this fraction of the budget. */
+export const COMPACT_KEEP_RATIO = 0.4;
+
+/** Minimum messages to keep hot (newest). */
+const MIN_KEEP_MESSAGES = 3;
+
+// ─── Consolidate ────────────────────────────────────────────────
+
+/**
+ * Check if consolidation should be triggered.
+ *
+ * @param {import('../conversation/persist.js').ConversationStore} conversationStore
+ * @param {number} [budget] — MESSAGE_TOKEN_BUDGET
+ * @returns {boolean}
+ */
+export function shouldConsolidate(conversationStore, budget = DEFAULT_MESSAGE_TOKEN_BUDGET) {
+  const hotTokens = conversationStore.hotTokens();
+  return hotTokens > budget;
+}
+
+/**
+ * Determine which messages to archive (move to cold).
+ * Strategy: from oldest, accumulate tokens until remaining ≤ budget * 40%.
+ * Always keep at least MIN_KEEP_MESSAGES.
+ *
+ * @param {object[]} messages — all hot messages, sorted chronologically
+ * @param {number} budget — MESSAGE_TOKEN_BUDGET
+ * @returns {{ toArchive: object[], toKeep: object[] }}
+ */
+export function partitionMessages(messages, budget = DEFAULT_MESSAGE_TOKEN_BUDGET) {
+  if (messages.length <= MIN_KEEP_MESSAGES) {
+    return { toArchive: [], toKeep: messages };
+  }
+
+  const keepBudget = Math.floor(budget * COMPACT_KEEP_RATIO);
+
+  // Work backwards from newest: accumulate tokens until we hit keepBudget
+  let keepTokens = 0;
+  let keepStart = messages.length;
+
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msgTokens = messages[i].tokens_est || 0;
+    if (keepTokens + msgTokens > keepBudget && (messages.length - i) >= MIN_KEEP_MESSAGES) {
+      keepStart = i + 1;
+      break;
+    }
+    keepTokens += msgTokens;
+    if (i === 0) keepStart = 0;
+  }
+
+  // Ensure at least MIN_KEEP_MESSAGES are kept
+  keepStart = Math.min(keepStart, messages.length - MIN_KEEP_MESSAGES);
+  keepStart = Math.max(keepStart, 0);
+
+  return {
+    toArchive: messages.slice(0, keepStart),
+    toKeep: messages.slice(keepStart),
+  };
+}
+
+/**
+ * Generate a compact summary of messages.
+ *
+ * @param {object[]} messages — messages to summarize
+ * @param {object} adapter — LLM adapter with .call()
+ * @param {object} config — { model }
+ * @returns {Promise<string>} — compact summary text
+ */
+async function generateSummary(messages, adapter, config) {
+  const conversation = messages.map(m => {
+    const prefix = m.role === 'user' ? 'User' : m.role === 'assistant' ? 'Assistant' : m.role;
+    return `[${prefix}]: ${(m.content || '').slice(0, 500)}`;
+  }).join('\n\n');
+
+  const system = 'You are a conversation summarizer. Summarize the conversation concisely in 2-3 paragraphs, preserving key decisions, facts, and context. Write in the same language as the conversation.';
+
+  try {
+    const result = await adapter.call({
+      model: config.model,
+      system,
+      messages: [{ role: 'user', content: `Summarize this conversation:\n\n${conversation}` }],
+      maxTokens: 1024,
+    });
+    return result.text.trim();
+  } catch {
+    // Fallback: simple concatenation of first/last messages
+    const first = messages[0]?.content?.slice(0, 200) || '';
+    const last = messages[messages.length - 1]?.content?.slice(0, 200) || '';
+    return `[Auto-summary failed] Started with: ${first}... Ended with: ${last}`;
+  }
+}
+
+/**
+ * Run the full Consolidate pipeline.
+ *
+ * 1. Partition messages (what to archive vs keep)
+ * 2. Generate compact summary (LLM call)
+ * 3. Extract memory entries (LLM call)
+ * 4. Move archived messages to cold/
+ * 5. Update compact.md, index.md, scopes.md
+ *
+ * @param {{
+ *   conversationStore: import('../conversation/persist.js').ConversationStore,
+ *   memoryStore: import('./store.js').MemoryStore,
+ *   adapter: object,
+ *   config: object,
+ *   budget?: number
+ * }} params
+ * @returns {Promise<{ compactSummary: string, extractedEntries: string[], archivedCount: number }>}
+ */
+export async function consolidate({ conversationStore, memoryStore, adapter, config, budget = DEFAULT_MESSAGE_TOKEN_BUDGET }) {
+  // Load all hot messages
+  const messages = conversationStore.loadAll();
+
+  if (messages.length <= MIN_KEEP_MESSAGES) {
+    return { compactSummary: '', extractedEntries: [], archivedCount: 0 };
+  }
+
+  // Step 1: Partition
+  const { toArchive, toKeep } = partitionMessages(messages, budget);
+
+  if (toArchive.length === 0) {
+    return { compactSummary: '', extractedEntries: [], archivedCount: 0 };
+  }
+
+  // Step 2: Generate compact summary
+  const compactSummary = await generateSummary(toArchive, adapter, config);
+
+  // Step 3: Extract memory entries
+  const extracted = await extractMemories({ messages: toArchive, adapter, config });
+
+  // Step 4: Move archived messages to cold
+  const archiveIds = toArchive.map(m => m.id).filter(Boolean);
+  conversationStore.moveToColdBatch(archiveIds);
+
+  // Step 5a: Update compact.md
+  if (compactSummary) {
+    conversationStore.updateCompactSummary(compactSummary);
+  }
+
+  // Step 5b: Write extracted memory entries
+  const entryNames = [];
+  for (const entry of extracted) {
+    const slug = memoryStore.writeEntry(entry);
+    entryNames.push(slug);
+  }
+
+  // Step 5c: Update index.md
+  const lastMsg = toKeep[toKeep.length - 1];
+  conversationStore.updateIndex({
+    lastMessageId: lastMsg?.id || null,
+  });
+
+  // Step 5d: Rebuild scopes.md
+  if (entryNames.length > 0) {
+    memoryStore.rebuildScopes();
+  }
+
+  return {
+    compactSummary,
+    extractedEntries: entryNames,
+    archivedCount: archiveIds.length,
+  };
+}