@hamp10/agentforge 0.1.0 → 0.1.1

package/src/OpenClawCLI.js

@@ -1,5 +1,5 @@
 import { spawn } from 'child_process';
-import { mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync } from 'fs';
+import { mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, unlinkSync } from 'fs';
 import { EventEmitter } from 'events';
 import { homedir } from 'os';
 import path from 'path';
@@ -40,6 +40,8 @@ export class OpenClawCLI extends EventEmitter {
     // OpenClaw Gateway streaming config — populated by worker.js on init
     this.gatewayPort = null;
     this.gatewayToken = null;
+    // AbortControllers for in-flight gateway requests — keyed by agentId
+    this.gatewayAbortControllers = new Map();
   }
 
   /**
@@ -52,6 +54,11 @@ export class OpenClawCLI extends EventEmitter {
     const url = `http://127.0.0.1:${this.gatewayPort}/v1/chat/completions`;
     const sessionKey = `agent:${agentId}:main`;
     let fullText = '';
+    const seenToolCallIds = new Set(); // also used in catch for timeout detection
+
+    // Register an AbortController so cancelAgent() can abort this fetch
+    const abortController = new AbortController();
+    this.gatewayAbortControllers.set(agentId, abortController);
 
     // Friendly names for tool calls shown as live messages to the user
     const toolLabels = {
@@ -67,6 +74,7 @@ export class OpenClawCLI extends EventEmitter {
     try {
       const res = await fetch(url, {
         method: 'POST',
+        signal: abortController.signal,
         headers: {
           'Authorization': `Bearer ${this.gatewayToken}`,
           'Content-Type': 'application/json',
@@ -79,7 +87,7 @@ export class OpenClawCLI extends EventEmitter {
           messages: [{ role: 'user', content: task }],
           stream: true,
         }),
-        signal: AbortSignal.timeout(600_000), // 10 min max
+        // No timeout — let agents run as long as needed
       });
       if (!res.ok) {
         console.warn(`[${agentId}] ⚠️ Streaming HTTP ${res.status} — falling back to subprocess`);
@@ -89,7 +97,7 @@ export class OpenClawCLI extends EventEmitter {
       // Parse SSE stream — emit text tokens AND tool call activity
       const decoder = new TextDecoder();
       let buffer = '';
-      const seenToolCallIds = new Set(); // avoid duplicate tool_activity for same call_id
+      // seenToolCallIds declared above try block
       const pendingToolCalls = new Map(); // index -> { id, name, args }
 
       for await (const rawChunk of res.body) {
@@ -159,8 +167,23 @@ export class OpenClawCLI extends EventEmitter {
         }
       }
       // Return an object so callers can distinguish "success with no text" from "request failed"
-      return { text: fullText, succeeded: true };
+      this.gatewayAbortControllers.delete(agentId);
+      return { text: fullText, succeeded: true, hadToolCalls: seenToolCallIds.size > 0 };
     } catch (err) {
+      this.gatewayAbortControllers.delete(agentId);
+      // Explicit cancel via cancelAgent() — treat as clean stop
+      if (err.name === 'AbortError') {
+        console.log(`[${agentId}] 🛑 Gateway request aborted by cancelAgent`);
+        return { text: fullText, succeeded: false, cancelled: true };
+      }
+      const isTimeout = err.name === 'TimeoutError' || err.message?.includes('timeout') || err.message?.includes('aborted');
+      if (isTimeout) {
+        console.warn(`[${agentId}] ⚠️ Streaming timed out after ${Math.round(fullText.length / 4)} tokens — gateway still processing`);
+        if (fullText.length > 0 || seenToolCallIds.size > 0) {
+          return { text: fullText, succeeded: true, hadToolCalls: seenToolCallIds.size > 0 };
+        }
+        return { text: '', succeeded: false, timedOut: true };
+      }
       console.warn(`[${agentId}] ⚠️ Streaming HTTP error: ${err.message} — falling back to subprocess`);
       return null; // null = request failed, subprocess fallback needed
     }
@@ -747,7 +770,7 @@ export class OpenClawCLI extends EventEmitter {
         // This ensures MEMORY.md and memory/ dir exist for memory persistence
         // Use bundled templates (packaged with worker) as primary source,
         // fall back to /tmp if somehow missing
-        const bundledTemplateDir = path.join(path.dirname(new URL(import.meta.url).pathname), '../../templates/agent');
+        const bundledTemplateDir = path.join(path.dirname(new URL(import.meta.url).pathname), '../templates/agent');
         const templateDir = existsSync(bundledTemplateDir) ? bundledTemplateDir : '/tmp/agentforge/templates/agent';
         console.log(`📁 Using templates from: ${templateDir}`);
         try {
@@ -800,7 +823,11 @@ export class OpenClawCLI extends EventEmitter {
    * Run an agent task
    * Images are saved to workspace and referenced in message for vision model analysis
    */
-  async runAgentTask(agentId, task, workDir, sessionId = null, image = null, browserProfile = null, imageWorkDir = null) {
+  async runAgentTask(agentId, task, workDir, sessionId = null, image = null, browserProfile = null, imageWorkDir = null, agentModel = null) {
+    // Apply per-agent model override before running (writes to openclaw.json + signals gateway)
+    if (agentModel) {
+      await this.setAgentModel(agentId, agentModel);
+    }
     // ── Gateway path disabled — subprocess shows live tool activity ──────────
     // Gateway path: SSE token streaming — tokens arrive live as the model generates.
     // Dashboard buffers tokens into sentences before showing each as a complete bubble.
@@ -808,20 +835,41 @@ export class OpenClawCLI extends EventEmitter {
       console.log(`\n🤖 Running agent (streaming): ${agentId}`);
       console.log(`   Task: ${task.slice(0, 120)}${task.length > 120 ? '…' : ''}`);
       try {
-        const streamResult = await this._runAgentTaskStreaming(agentId, task, sessionId);
+        // Retry up to 3 times on empty responses (API overloaded/rate limit)
+        let streamResult = null;
+        for (let attempt = 1; attempt <= 3; attempt++) {
+          streamResult = await this._runAgentTaskStreaming(agentId, task, sessionId);
+          if (streamResult === null) break; // connection failure — fall back to subprocess
+          if (streamResult.timedOut) break; // timeout — don't retry
+          if (streamResult.text || streamResult.hadToolCalls) break; // got real output — done
+          // Empty response — API overloaded, wait and retry
+          console.warn(`[${agentId}] ⚠️ Gateway returned empty response (attempt ${attempt}/3) — retrying in ${attempt * 5}s`);
+          if (attempt < 3) await new Promise(r => setTimeout(r, attempt * 5000));
+        }
         if (streamResult !== null) {
+          // Streaming timed out with zero output — gateway still processing, can't fall back
+          if (streamResult.timedOut) {
+            const errorText = "Agent timed out: the model took too long to respond. The gateway is still processing — please wait a moment and try again.";
+            let identity = { identityName: agentId, identityEmoji: '🤖' };
+            try { identity = await this.getAgentIdentity(agentId); } catch { /* ignore */ }
+            this.emit('agent_completed', { agentId, duration: 0, result: { output: errorText }, identity });
+            return { success: true, agentId, duration: 0, result: { output: errorText }, identity };
+          }
           // Gateway request succeeded (streamResult.succeeded === true).
-          // Use the text response if any; if the agent only did tool work with no
-          // text output, return empty string — do NOT fall back to subprocess (which
-          // would re-run the same task a second time and corrupt the session state).
           const responseText = streamResult.text || '';
-          if (!responseText) {
-            console.log(`[${agentId}] ✅ Gateway task completed with no text output (tool-only task)`);
+          if (!responseText && !streamResult.hadToolCalls) {
+            // Still empty after retries — API is down, fall back to subprocess
+            console.warn(`[${agentId}] ⚠️ Gateway returned no content after 3 attempts — falling back to subprocess`);
+            // fall through to subprocess path below
+          } else {
+            if (!responseText) {
+              console.log(`[${agentId}] ✅ Gateway task completed with no text output (tool-only task)`);
+            }
+            let identity = { identityName: agentId, identityEmoji: '🤖' };
+            try { identity = await this.getAgentIdentity(agentId); } catch { /* ignore */ }
+            this.emit('agent_completed', { agentId, duration: 0, result: { output: responseText }, identity });
+            return { success: true, agentId, duration: 0, result: { output: responseText }, identity };
           }
-          let identity = { identityName: agentId, identityEmoji: '🤖' };
-          try { identity = await this.getAgentIdentity(agentId); } catch { /* ignore */ }
-          this.emit('agent_completed', { agentId, duration: 0, result: { output: responseText }, identity });
-          return { success: true, agentId, duration: 0, result: { output: responseText }, identity };
         }
         console.warn(`[${agentId}] ⚠️ Streaming request failed — falling back to subprocess`);
       } catch (err) {
@@ -956,18 +1004,40 @@ export class OpenClawCLI extends EventEmitter {
       const existingAgent = this.activeAgents.get(agentId);
       if (existingAgent && existingAgent.proc && !existingAgent.proc.killed) {
         console.log(`[${agentId}] 🔪 Killing lingering process (pid ${existingAgent.proc.pid}) before spawning new one`);
+        const lingerPgid = existingAgent.pgid || existingAgent.proc.pid;
+        try { process.kill(-lingerPgid, 'SIGKILL'); } catch (e) { /* already dead */ }
         try { treeKill(existingAgent.proc.pid, 'SIGKILL'); } catch (e) { /* already dead */ }
         this.activeAgents.delete(agentId);
         // Wait for process to fully exit and release file locks before spawning
         await new Promise(r => setTimeout(r, 800));
       }
 
+      // Nuke any stale lock files before spawning — the gateway (a persistent process)
+      // can hold session locks indefinitely after a killed task and never release them,
+      // causing the next task to fail with "session file locked".
+      const sessionDir = path.join(homedir(), '.openclaw', 'agents', agentId, 'sessions');
+      if (existsSync(sessionDir)) {
+        try {
+          for (const f of readdirSync(sessionDir)) {
+            if (f.endsWith('.lock')) {
+              unlinkSync(path.join(sessionDir, f));
+              console.log(`[${agentId}] 🔓 Cleared stale lock before spawn: ${f}`);
+            }
+          }
+        } catch (e) { /* ignore */ }
+      }
+
       // Change to working directory and run agent
       // Use process.execPath (node) directly to avoid shell metacharacter issues
       // with user message content (quotes, apostrophes, etc.)
+      // detached: true makes the child a process group leader (pgid = proc.pid).
+      // Background subprocesses spawned by openclaw's exec tool inherit this pgid
+      // (bash disables job control when non-interactive, so & doesn't create new groups).
+      // This lets cancelAgent kill the entire group — including orphaned bg processes.
       const proc = spawn(process.execPath, [this.bin, ...args], {
         cwd: workDir,
-        env: agentEnv
+        env: agentEnv,
+        detached: true
       });
 
       let output = '';
@@ -980,7 +1050,7 @@ export class OpenClawCLI extends EventEmitter {
       const firstOutputTimer = setTimeout(() => {
         if (!firstOutputSeen && !promiseSettled) {
           console.warn(`[${agentId}] ⚠️ No output in 90s — openclaw hung, killing`);
-          try { proc.kill('SIGKILL'); } catch (e) { /* already dead */ }
+          treeKill(proc.pid, 'SIGKILL'); // kill entire process tree — sub-agents included
           if (!promiseSettled) {
             promiseSettled = true;
             this.activeAgents.delete(agentId);
@@ -1014,8 +1084,7 @@ export class OpenClawCLI extends EventEmitter {
                 if (runCompleted || promiseSettled) return; // already handled
                 console.log(`[${agentId}] ⚠️ Process still running 30s after agent end — force killing (compaction hung?)`);
                 runCompleted = true;
-                try { proc.kill('SIGTERM'); } catch (e) { /* already dead */ }
-                setTimeout(() => { try { proc.kill('SIGKILL'); } catch (e) {} }, 1000);
+                treeKill(proc.pid, 'SIGKILL'); // kill entire process tree — sub-agents included
                 if (!promiseSettled) {
                   promiseSettled = true;
                   const duration = Date.now() - startTime;
@@ -1050,16 +1119,7 @@ export class OpenClawCLI extends EventEmitter {
               completionTimer = setTimeout(async () => {
                 if (proc && !proc.killed) {
                   console.log(`[${agentId}] ⚠️ Process didn't exit after run completed, force killing`);
-                  try {
-                    proc.kill('SIGTERM');
-                    setTimeout(() => {
-                      if (!proc.killed) {
-                        proc.kill('SIGKILL');
-                      }
-                    }, 1000);
-                  } catch (e) {
-                    // Process might already be dead
-                  }
+                  treeKill(proc.pid, 'SIGKILL'); // kill entire process tree — sub-agents included
                 }
                 // Task completed successfully — resolve now instead of waiting for close event.
                 // The close event can hang indefinitely if openclaw's child subprocesses keep
@@ -1199,6 +1259,8 @@ export class OpenClawCLI extends EventEmitter {
       proc.on('close', async (code) => {
         const duration = Date.now() - startTime;
         clearTimeout(firstOutputTimer);
+        // Sweep up any surviving child processes (e.g. sessions_spawn sub-agents that outlived the parent)
+        try { treeKill(proc.pid, 'SIGKILL'); } catch (e) { /* already dead */ }
 
         // Clear the completion timer if it's still running
         if (completionTimer) {
@@ -1278,12 +1340,13 @@ export class OpenClawCLI extends EventEmitter {
         }
       });
 
-      // Track active agent
+      // Track active agent (pgid = proc.pid because detached:true makes it a group leader)
       this.activeAgents.set(agentId, {
         proc,
         startTime,
         task,
-        workDir
+        workDir,
+        pgid: proc.pid
       });
     });
   }
@@ -1320,6 +1383,73 @@ export class OpenClawCLI extends EventEmitter {
     return results;
   }
 
+  /**
+   * Set per-agent model override in ~/.openclaw/openclaw.json
+   * Adds the model to the catalog allowlist and sets the agent's primary model.
+   * Called before running a task so the spawned openclaw process (or gateway) picks it up.
+   */
+  async setAgentModel(agentId, modelString) {
+    if (!modelString) return;
+    const cfgPath = path.join(homedir(), '.openclaw', 'openclaw.json');
+    try {
+      let cfg = {};
+      if (existsSync(cfgPath)) {
+        cfg = JSON.parse(readFileSync(cfgPath, 'utf-8'));
+      }
+      // Ensure structure
+      if (!cfg.agents) cfg.agents = {};
+      if (!cfg.agents.defaults) cfg.agents.defaults = {};
+      if (!cfg.agents.defaults.models) cfg.agents.defaults.models = {};
+      if (!cfg.agents.list) cfg.agents.list = [];
+
+      // Add to catalog/allowlist so OpenClaw permits this model
+      if (!cfg.agents.defaults.models[modelString]) {
+        cfg.agents.defaults.models[modelString] = {};
+      }
+
+      // Find or create per-agent entry
+      let agentEntry = cfg.agents.list.find(a => a.id === agentId);
+      if (!agentEntry) {
+        agentEntry = { id: agentId };
+        cfg.agents.list.push(agentEntry);
+      }
+
+      // Set per-agent model override
+      agentEntry.model = { primary: modelString };
+
+      writeFileSync(cfgPath, JSON.stringify(cfg, null, 2));
+      console.log(`[${agentId}] 🔧 Model override → ${modelString}`);
+
+      // Signal gateway to reload config so streaming path picks up the new model
+      if (this.gatewayPort && this.gatewayToken) {
+        try {
+          // Try PATCH /api/config (OpenClaw gateway REST endpoint)
+          const res = await fetch(`http://127.0.0.1:${this.gatewayPort}/api/config`, {
+            method: 'PATCH',
+            headers: {
+              'Authorization': `Bearer ${this.gatewayToken}`,
+              'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+              agents: {
+                defaults: { models: cfg.agents.defaults.models },
+                list: cfg.agents.list,
+              },
+            }),
+            signal: AbortSignal.timeout(2000),
+          });
+          if (res.ok) {
+            console.log(`[${agentId}] ✅ Gateway config patched`);
+          }
+        } catch {
+          // Not fatal — subprocess path reads config fresh each time
+        }
+      }
+    } catch (err) {
+      console.warn(`[${agentId}] ⚠️ setAgentModel failed: ${err.message}`);
+    }
+  }
+
   /**
    * List all agents (with timeout to prevent hanging)
    */
@@ -1490,27 +1620,61 @@ export class OpenClawCLI extends EventEmitter {
    * Cancel a running agent task by killing the process tree immediately
    */
   cancelAgent(agentId) {
+    // Kill gateway fetch if running in streaming mode
+    const abortCtrl = this.gatewayAbortControllers.get(agentId);
+    if (abortCtrl) {
+      console.log(`🛑 Aborting gateway stream for agent ${agentId}`);
+      abortCtrl.abort();
+      this.gatewayAbortControllers.delete(agentId);
+    }
+
     const agentInfo = this.activeAgents.get(agentId);
     if (!agentInfo || !agentInfo.proc) {
+      if (abortCtrl) return true; // gateway abort counts as success
       console.log(`⚠️  No running process found for agent ${agentId}`);
       return false;
     }
 
-    const { proc } = agentInfo;
+    const { proc, pgid } = agentInfo;
     const pid = proc.pid;
-    
-    console.log(`🛑 Killing process tree for agent ${agentId} (PID: ${pid})`);
-    
+
+    console.log(`🛑 Killing process group for agent ${agentId} (PID: ${pid}, PGID: ${pgid || pid})`);
+
     // Clean up tracking immediately
     this.activeAgents.delete(agentId);
-    
-    // Use tree-kill to kill the entire process tree with SIGKILL (immediate, no grace period)
+
+    // Kill the entire process GROUP first (catches background/orphaned subprocesses that
+    // escaped the process tree, e.g. `python3 -m http.server &` spawned by openclaw exec).
+    // Since we spawn with detached:true, pgid == proc.pid and all non-interactive bash
+    // background processes inherit this pgid.
+    const groupId = pgid || pid;
+    try {
+      process.kill(-groupId, 'SIGKILL');
+      console.log(`✅ Process group -${groupId} killed`);
+    } catch (e) {
+      // Group may already be dead or process detached before we set pgid
+      console.log(`⚠️  process group kill (-${groupId}): ${e.message}`);
+    }
+
+    // Also use tree-kill as belt-and-suspenders for any sub-processes that changed pgid
     treeKill(pid, 'SIGKILL', (err) => {
       if (err) {
         console.log(`⚠️  tree-kill error (process may already be dead): ${err.message}`);
       } else {
         console.log(`✅ Process tree ${pid} killed successfully`);
       }
+      // Clean up any stale lock files left by the killed process
+      const sessionDir = path.join(homedir(), '.openclaw', 'agents', agentId, 'sessions');
+      if (existsSync(sessionDir)) {
+        try {
+          for (const f of readdirSync(sessionDir)) {
+            if (f.endsWith('.lock')) {
+              unlinkSync(path.join(sessionDir, f));
+              console.log(`🔓 Removed stale lock: ${f}`);
+            }
+          }
+        } catch (e) { /* ignore */ }
+      }
     });
     
     this.emit('agent_cancelled', { agentId });