@hamp10/agentforge 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/agentforge.js +353 -44
- package/package.json +1 -1
- package/src/OpenClawCLI.js +202 -38
- package/src/worker.js +260 -226
- package/templates/agent/AGENTFORGE.md +148 -56
- package/templates/agent/AGENTS.md +0 -212
- package/templates/agent/SOUL.md +0 -36
- package/templates/agent/TOOLS.md +0 -40
package/src/worker.js
CHANGED
|
@@ -6,7 +6,7 @@ import { HampAgentCLI } from './HampAgentCLI.js';
|
|
|
6
6
|
import { OllamaAgent } from './OllamaAgent.js';
|
|
7
7
|
import EventEmitter from 'events';
|
|
8
8
|
import path from 'path';
|
|
9
|
-
import { existsSync, readdirSync, readFileSync, mkdirSync, writeFileSync, copyFileSync, statSync, unlinkSync } from 'fs';
|
|
9
|
+
import { existsSync, readdirSync, readFileSync, mkdirSync, writeFileSync, copyFileSync, statSync, unlinkSync, openSync } from 'fs';
|
|
10
10
|
import { fileURLToPath } from 'url';
|
|
11
11
|
import { homedir, hostname } from 'os';
|
|
12
12
|
import { spawn } from 'child_process';
|
|
@@ -71,9 +71,17 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
71
71
|
this.agentProcessing = new Map(); // agentId -> boolean (is currently processing)
|
|
72
72
|
this.processingStartTime = new Map(); // agentId -> timestamp when processing started
|
|
73
73
|
this.PROCESSING_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes max for stale processing state (large projects with Opus can be slow)
|
|
74
|
+
|
|
75
|
+
// Browser mutex — only one agent can use the browser (CDP) at a time.
|
|
76
|
+
// Others wait in a queue. Prevents parallel agents from conflicting on port 9223.
|
|
77
|
+
this._browserQueue = [];
|
|
78
|
+
this._browserBusy = false;
|
|
74
79
|
|
|
75
80
|
// Track running tasks for cancellation
|
|
76
81
|
this.runningTasks = new Map(); // taskId -> { agentId, cancelled }
|
|
82
|
+
|
|
83
|
+
// Track last output time per agent — used to detect broken gateway streams after reconnect
|
|
84
|
+
this.lastOutputTime = new Map(); // agentId -> timestamp
|
|
77
85
|
|
|
78
86
|
// Queue for messages that couldn't be sent while disconnected
|
|
79
87
|
this.pendingMessages = [];
|
|
@@ -83,10 +91,6 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
83
91
|
this.recentCompletions = new Set();
|
|
84
92
|
this.completionTTL = 30000; // 30 seconds
|
|
85
93
|
|
|
86
|
-
// Track agent activity for stuck detection
|
|
87
|
-
this.lastAgentActivity = new Map(); // agentId -> timestamp
|
|
88
|
-
this.pingsSinceActivity = new Map(); // agentId -> count
|
|
89
|
-
this.STUCK_PING_THRESHOLD = 2; // 2 pings with no activity = stuck (~60s since server pings every 30s)
|
|
90
94
|
}
|
|
91
95
|
|
|
92
96
|
speakTextOutLoud(utterance) {
|
|
@@ -170,17 +174,42 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
170
174
|
|
|
171
175
|
async initialize() {
|
|
172
176
|
this._killOrphanedAgents();
|
|
177
|
+
await this._startGateway();
|
|
173
178
|
this.installPreviewServer();
|
|
174
179
|
this._startAutoUpdateCheck();
|
|
175
180
|
console.log('✅ Worker initialized');
|
|
176
181
|
}
|
|
177
182
|
|
|
183
|
+
async _startGateway() {
|
|
184
|
+
// Spawn openclaw-gateway as a child of this worker process.
|
|
185
|
+
// When the worker exits (Ctrl+C in terminal), the gateway dies with it.
|
|
186
|
+
// No LaunchAgent needed — the terminal session owns everything.
|
|
187
|
+
const openclaw = process.env.OPENCLAW_PATH ||
|
|
188
|
+
'/Users/hamp/.npm-global/lib/node_modules/openclaw/dist/index.js';
|
|
189
|
+
const port = 18789;
|
|
190
|
+
const logDir = path.join(homedir(), '.openclaw', 'logs');
|
|
191
|
+
mkdirSync(logDir, { recursive: true });
|
|
192
|
+
const logOut = openSync(path.join(logDir, 'gateway.log'), 'a');
|
|
193
|
+
const logErr = openSync(path.join(logDir, 'gateway.err.log'), 'a');
|
|
194
|
+
const gw = spawn(process.execPath, [openclaw, 'gateway', '--port', String(port)], {
|
|
195
|
+
stdio: ['ignore', logOut, logErr],
|
|
196
|
+
env: { ...process.env, NODE_EXTRA_CA_CERTS: '/etc/ssl/cert.pem' },
|
|
197
|
+
detached: false
|
|
198
|
+
});
|
|
199
|
+
gw.on('exit', (code) => {
|
|
200
|
+
if (code !== null) console.warn(`⚠️ openclaw-gateway exited (code ${code}) — browser tools will fail until worker restarts`);
|
|
201
|
+
});
|
|
202
|
+
console.log(`🌐 OpenClaw Gateway started (PID: ${gw.pid}, port: ${port})`);
|
|
203
|
+
// Brief pause so gateway is listening before first agent task
|
|
204
|
+
await new Promise(r => setTimeout(r, 1500));
|
|
205
|
+
}
|
|
206
|
+
|
|
178
207
|
_killOrphanedAgents() {
|
|
179
208
|
// Kill any openclaw agent processes left over from a previous worker session.
|
|
180
209
|
// Without this, orphaned processes reconnect to the gateway and block the task queue.
|
|
181
210
|
for (const name of ['openclaw-agent', 'openclaw-gateway']) {
|
|
182
211
|
try {
|
|
183
|
-
const p = spawn('pkill', ['-f', name], { stdio: 'ignore' });
|
|
212
|
+
const p = spawn('pkill', ['-9', '-f', name], { stdio: 'ignore' });
|
|
184
213
|
p.on('close', (code) => {
|
|
185
214
|
if (code === 0) console.log(`🧹 Killed orphaned ${name} processes`);
|
|
186
215
|
});
|
|
@@ -323,6 +352,37 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
323
352
|
this.flushPendingMessages();
|
|
324
353
|
this.processAllQueues(); // Kick-start any stalled queues
|
|
325
354
|
}, 500);
|
|
355
|
+
|
|
356
|
+
// After 90s, cancel any running tasks that produced zero output since reconnect.
|
|
357
|
+
// This catches broken gateway streams that went dark during the disconnect.
|
|
358
|
+
// A healthy agent mid-build produces output continuously (tool calls, writes, etc.)
|
|
359
|
+
// and never goes 90s silent. Only broken/frozen streams stay silent this long.
|
|
360
|
+
const tasksAtReconnect = new Map(this.runningTasks);
|
|
361
|
+
const reconnectTime = Date.now();
|
|
362
|
+
setTimeout(() => {
|
|
363
|
+
for (const [tid, taskInfo] of tasksAtReconnect.entries()) {
|
|
364
|
+
if (taskInfo.cancelled) continue;
|
|
365
|
+
const current = this.runningTasks.get(tid);
|
|
366
|
+
if (!current || current.cancelled) continue; // task finished normally
|
|
367
|
+
const lastOut = this.lastOutputTime.get(taskInfo.agentId) || 0;
|
|
368
|
+
if (lastOut < reconnectTime) {
|
|
369
|
+
// No output since the reconnect — stream is dead
|
|
370
|
+
console.log(`⚠️ Agent ${taskInfo.agentId} produced no output in 45s after reconnect — cancelling (broken stream)`);
|
|
371
|
+
this.cli.cancelAgent(taskInfo.agentId);
|
|
372
|
+
this.runningTasks.delete(tid);
|
|
373
|
+
this.agentProcessing.set(taskInfo.agentId, false);
|
|
374
|
+
this.processingStartTime.delete(taskInfo.agentId);
|
|
375
|
+
this.send({
|
|
376
|
+
type: 'task_progress',
|
|
377
|
+
taskId: tid,
|
|
378
|
+
agentId: taskInfo.agentId,
|
|
379
|
+
output: '⚠️ Task was interrupted by a connection issue. Please resend your message to continue.',
|
|
380
|
+
isChunk: true
|
|
381
|
+
});
|
|
382
|
+
this.send({ type: 'task_cancelled', taskId: tid, agentId: taskInfo.agentId });
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}, 90000);
|
|
326
386
|
}
|
|
327
387
|
|
|
328
388
|
resolve();
|
|
@@ -395,22 +455,34 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
395
455
|
await this.executeTask(message);
|
|
396
456
|
break;
|
|
397
457
|
|
|
398
|
-
case 'task_cancel':
|
|
399
|
-
|
|
400
|
-
console.log(`📨 CANCEL REQUEST: taskId=${
|
|
401
|
-
if (
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
await this.cancelTaskByAgent(message.agentId);
|
|
405
|
-
} else {
|
|
406
|
-
console.log(`⚠️ task_cancel received without taskId or agentId!`);
|
|
458
|
+
case 'task_cancel': {
|
|
459
|
+
const { agentId: cancelAgentId, taskId: cancelTaskId } = message;
|
|
460
|
+
console.log(`📨 CANCEL REQUEST: taskId=${cancelTaskId} agentId=${cancelAgentId}`);
|
|
461
|
+
if (!cancelAgentId) {
|
|
462
|
+
console.log(`⚠️ task_cancel received without agentId!`);
|
|
463
|
+
break;
|
|
407
464
|
}
|
|
465
|
+
// Always kill the process immediately by agentId — don't wait for task lookup
|
|
466
|
+
// Task lookup can fail if taskId is stale (agent moved to a queued message)
|
|
467
|
+
const killed = this.cli.cancelAgent(cancelAgentId) || this.hampagent?.cancelAgent(cancelAgentId) || false;
|
|
468
|
+
console.log(`🛑 Direct kill for agent ${cancelAgentId}: ${killed}`);
|
|
469
|
+
// Clean up all state for this agent
|
|
470
|
+
this.agentQueues.set(cancelAgentId, []);
|
|
471
|
+
this.agentProcessing.set(cancelAgentId, false);
|
|
472
|
+
this.processingStartTime.delete(cancelAgentId);
|
|
473
|
+
// Mark any tracked tasks for this agent as cancelled
|
|
474
|
+
for (const [tid, info] of this.runningTasks.entries()) {
|
|
475
|
+
if (info.agentId === cancelAgentId) {
|
|
476
|
+
info.cancelled = true;
|
|
477
|
+
this.runningTasks.delete(tid);
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
this.send({ type: 'task_cancelled', taskId: cancelTaskId, agentId: cancelAgentId });
|
|
408
481
|
break;
|
|
482
|
+
}
|
|
409
483
|
|
|
410
484
|
case 'ping':
|
|
411
485
|
this.send({ type: 'pong' });
|
|
412
|
-
// Check for stuck agents - if processing but no activity for 2+ pings
|
|
413
|
-
this.checkForStuckAgents();
|
|
414
486
|
break;
|
|
415
487
|
|
|
416
488
|
case 'worker_restart':
|
|
@@ -440,6 +512,47 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
440
512
|
break;
|
|
441
513
|
}
|
|
442
514
|
|
|
515
|
+
case 'get_models': {
|
|
516
|
+
// Return locally available Ollama models + openclaw.json catalog
|
|
517
|
+
let ollamaModels = [];
|
|
518
|
+
let catalogModels = {};
|
|
519
|
+
try {
|
|
520
|
+
const resp = await fetch('http://localhost:11434/api/tags', {
|
|
521
|
+
signal: AbortSignal.timeout(3000),
|
|
522
|
+
});
|
|
523
|
+
if (resp.ok) {
|
|
524
|
+
const data = await resp.json();
|
|
525
|
+
ollamaModels = (data.models || []).map(m => ({
|
|
526
|
+
id: `ollama/${m.name}`,
|
|
527
|
+
name: m.name,
|
|
528
|
+
size: m.size,
|
|
529
|
+
tier: 'local',
|
|
530
|
+
}));
|
|
531
|
+
}
|
|
532
|
+
} catch {
|
|
533
|
+
// Ollama not running — that's fine
|
|
534
|
+
}
|
|
535
|
+
let primaryModel = null;
|
|
536
|
+
try {
|
|
537
|
+
const cfgPath = path.join(homedir(), '.openclaw', 'openclaw.json');
|
|
538
|
+
if (existsSync(cfgPath)) {
|
|
539
|
+
const cfg = JSON.parse(readFileSync(cfgPath, 'utf-8'));
|
|
540
|
+
catalogModels = cfg?.agents?.defaults?.models || {};
|
|
541
|
+
primaryModel = cfg?.agents?.defaults?.model?.primary || null;
|
|
542
|
+
}
|
|
543
|
+
} catch {
|
|
544
|
+
// ignore
|
|
545
|
+
}
|
|
546
|
+
this.send({
|
|
547
|
+
type: 'get_models_result',
|
|
548
|
+
requestId: message.requestId,
|
|
549
|
+
ollamaModels,
|
|
550
|
+
catalogModels,
|
|
551
|
+
primaryModel,
|
|
552
|
+
});
|
|
553
|
+
break;
|
|
554
|
+
}
|
|
555
|
+
|
|
443
556
|
default:
|
|
444
557
|
console.log(`⚠️ Unknown message type: ${message.type}`);
|
|
445
558
|
}
|
|
@@ -470,6 +583,31 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
470
583
|
this.processQueue(agentId);
|
|
471
584
|
}
|
|
472
585
|
|
|
586
|
+
// Browser mutex — serialises all browser tool access across parallel agents.
|
|
587
|
+
// Returns a release function; call it when the agent is done with the browser.
|
|
588
|
+
acquireBrowser(agentId) {
|
|
589
|
+
return new Promise((resolve) => {
|
|
590
|
+
const attempt = () => {
|
|
591
|
+
if (!this._browserBusy) {
|
|
592
|
+
this._browserBusy = true;
|
|
593
|
+
console.log(`🌐 [${agentId}] Acquired browser`);
|
|
594
|
+
resolve(() => {
|
|
595
|
+
console.log(`🌐 [${agentId}] Released browser`);
|
|
596
|
+
this._browserBusy = false;
|
|
597
|
+
if (this._browserQueue.length > 0) {
|
|
598
|
+
const next = this._browserQueue.shift();
|
|
599
|
+
next();
|
|
600
|
+
}
|
|
601
|
+
});
|
|
602
|
+
} else {
|
|
603
|
+
console.log(`🌐 [${agentId}] Waiting for browser (${this._browserQueue.length + 1} in queue)`);
|
|
604
|
+
this._browserQueue.push(attempt);
|
|
605
|
+
}
|
|
606
|
+
};
|
|
607
|
+
attempt();
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
|
|
473
611
|
async processQueue(agentId) {
|
|
474
612
|
// If already processing, check if it's stale
|
|
475
613
|
if (this.agentProcessing.get(agentId)) {
|
|
@@ -498,8 +636,9 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
498
636
|
}
|
|
499
637
|
|
|
500
638
|
// Mark as processing with timestamp
|
|
639
|
+
const myStartTime = Date.now();
|
|
501
640
|
this.agentProcessing.set(agentId, true);
|
|
502
|
-
this.processingStartTime.set(agentId,
|
|
641
|
+
this.processingStartTime.set(agentId, myStartTime);
|
|
503
642
|
console.log(`🚀 Starting task for ${agentId} (${queue.length} in queue)`);
|
|
504
643
|
|
|
505
644
|
// Get next task from queue
|
|
@@ -584,10 +723,14 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
584
723
|
if (queueTimeoutFired && !executeTaskCompleted) {
|
|
585
724
|
console.log(`[${agentId}] ⚠️ Queue timeout won the race - executeTaskNow never completed`);
|
|
586
725
|
}
|
|
587
|
-
//
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
726
|
+
// Only clear processing state if we still own it (a newer task may have taken over)
|
|
727
|
+
if (this.processingStartTime.get(agentId) === myStartTime) {
|
|
728
|
+
console.log(`🧹 Clearing processing state for ${agentId}`);
|
|
729
|
+
this.agentProcessing.set(agentId, false);
|
|
730
|
+
this.processingStartTime.delete(agentId);
|
|
731
|
+
} else {
|
|
732
|
+
console.log(`🧹 Skipping processing state clear for ${agentId} — newer task owns it`);
|
|
733
|
+
}
|
|
591
734
|
|
|
592
735
|
// Process next task if queue is not empty
|
|
593
736
|
if (queue.length > 0) {
|
|
@@ -604,7 +747,7 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
604
747
|
}
|
|
605
748
|
}
|
|
606
749
|
|
|
607
|
-
async executeTaskNow({ taskId, agentId, sessionId, message: userMessage, workDir, defaultProjectsPath, image, roomId, roomContext, isMaestro, conversationHistory, browserProfile, agentName, agentEmoji, runnerType }) {
|
|
750
|
+
async executeTaskNow({ taskId, agentId, sessionId, message: userMessage, workDir, defaultProjectsPath, image, roomId, roomContext, isMaestro, conversationHistory, browserProfile, agentName, agentEmoji, runnerType, agentModel }) {
|
|
608
751
|
const isMaestroTask = isMaestro || agentId === 'maestro';
|
|
609
752
|
console.log(`🤖 Executing task ${taskId} for agent ${agentId}${isMaestroTask ? ' (MAESTRO)' : ''}${browserProfile ? ` [browser: ${browserProfile}]` : ''}`);
|
|
610
753
|
if (sessionId) {
|
|
@@ -713,9 +856,8 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
713
856
|
if (!taskInfo || taskInfo.cancelled) {
|
|
714
857
|
return; // Task cancelled, drop all output
|
|
715
858
|
}
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
this.recordAgentActivity(agentId);
|
|
859
|
+
// Track last output time for broken-stream detection after reconnect
|
|
860
|
+
this.lastOutputTime.set(agentId, Date.now());
|
|
719
861
|
|
|
720
862
|
// Filter out tool error stack traces from room chat
|
|
721
863
|
const text = data.output?.trim();
|
|
@@ -827,11 +969,8 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
827
969
|
// Set up tool activity streaming (shows what tool agent is using)
|
|
828
970
|
const toolActivityHandler = (data) => {
|
|
829
971
|
if (data.agentId === agentId) {
|
|
830
|
-
// Record activity to prevent stuck detection from firing
|
|
831
|
-
this.recordAgentActivity(agentId);
|
|
832
|
-
|
|
833
972
|
// tts tool is handled natively by openclaw — do not intercept or emit anything
|
|
834
|
-
|
|
973
|
+
|
|
835
974
|
let toolInputPreview;
|
|
836
975
|
if (data.toolInput) {
|
|
837
976
|
try {
|
|
@@ -841,6 +980,11 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
841
980
|
}
|
|
842
981
|
}
|
|
843
982
|
|
|
983
|
+
// Log tool calls to worker.log for always-on visibility
|
|
984
|
+
if ((data.event === 'tool_start' || data.event === 'start') && data.description) {
|
|
985
|
+
console.log(`[${agentId}] 🔧 ${data.description}${toolInputPreview ? ` — ${toolInputPreview.slice(0, 120)}` : ''}`);
|
|
986
|
+
}
|
|
987
|
+
|
|
844
988
|
this.send({
|
|
845
989
|
type: 'tool_activity',
|
|
846
990
|
taskId,
|
|
@@ -858,69 +1002,10 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
858
1002
|
activeRunner.on('agent_error', errorHandler);
|
|
859
1003
|
activeRunner.on('tool_activity', toolActivityHandler);
|
|
860
1004
|
|
|
861
|
-
// Listen for raw alive signals (any stdout, even filtered) to prevent false stuck detection
|
|
862
|
-
const aliveHandler = (data) => {
|
|
863
|
-
if (data.agentId === agentId) {
|
|
864
|
-
this.recordAgentActivity(agentId);
|
|
865
|
-
}
|
|
866
|
-
};
|
|
867
|
-
activeRunner.on('agent_alive', aliveHandler);
|
|
868
|
-
|
|
869
|
-
// Inactivity: warn at 60s, KILL at 10 minutes of silence (no stdout at all)
|
|
870
|
-
const INACTIVITY_WARN_MS = 60000;
|
|
871
|
-
const INACTIVITY_KILL_MS = 2 * 60 * 1000; // 2 minutes — kill truly hung openclaw
|
|
872
|
-
let lastActivityTime = Date.now();
|
|
873
|
-
let inactivityTimer = null;
|
|
874
|
-
let inactivityKillTimer = null;
|
|
875
1005
|
let currentTool = null; // Track which tool is currently running
|
|
876
|
-
let promiseSettled = false;
|
|
877
|
-
|
|
878
|
-
// Tools that are expected to take a while - don't warn about these
|
|
879
|
-
const QUIET_TOOLS = ['Editing file', 'Writing file', 'Reading file', 'edit', 'write', 'read'];
|
|
880
|
-
|
|
881
|
-
const clearInactivityTimers = () => {
|
|
882
|
-
if (inactivityTimer) { clearTimeout(inactivityTimer); inactivityTimer = null; }
|
|
883
|
-
if (inactivityKillTimer) { clearTimeout(inactivityKillTimer); inactivityKillTimer = null; }
|
|
884
|
-
};
|
|
1006
|
+
let promiseSettled = false;
|
|
885
1007
|
|
|
886
1008
|
const resetInactivityTimer = () => {
|
|
887
|
-
lastActivityTime = Date.now();
|
|
888
|
-
// Also reset the ping-based stuck detector so 300s kill doesn't fire during active work
|
|
889
|
-
this.lastAgentActivity.set(agentId, Date.now());
|
|
890
|
-
this.pingsSinceActivity.set(agentId, 0);
|
|
891
|
-
clearInactivityTimers();
|
|
892
|
-
|
|
893
|
-
// Warn at 30s
|
|
894
|
-
inactivityTimer = setTimeout(() => {
|
|
895
|
-
const taskInfo = this.runningTasks.get(taskId);
|
|
896
|
-
if (taskInfo && !taskInfo.cancelled) {
|
|
897
|
-
if (currentTool && QUIET_TOOLS.some(t => currentTool.toLowerCase().includes(t.toLowerCase()))) {
|
|
898
|
-
resetInactivityTimer(); // quiet tool — just reset and check again later
|
|
899
|
-
return;
|
|
900
|
-
}
|
|
901
|
-
const stuckTool = currentTool ? ` while running "${currentTool}"` : '';
|
|
902
|
-
this.send({
|
|
903
|
-
type: 'task_warning',
|
|
904
|
-
taskId,
|
|
905
|
-
agentId,
|
|
906
|
-
roomId,
|
|
907
|
-
warning: `No activity for ${INACTIVITY_WARN_MS/1000} seconds${stuckTool} - agent may be stuck`,
|
|
908
|
-
lastTool: currentTool
|
|
909
|
-
});
|
|
910
|
-
}
|
|
911
|
-
}, INACTIVITY_WARN_MS);
|
|
912
|
-
|
|
913
|
-
// Kill at 10 minutes — openclaw is definitely hung
|
|
914
|
-
inactivityKillTimer = setTimeout(() => {
|
|
915
|
-
if (promiseSettled) return;
|
|
916
|
-
const taskInfo = this.runningTasks.get(taskId);
|
|
917
|
-
if (taskInfo && !taskInfo.cancelled) {
|
|
918
|
-
console.warn(`[${agentId}] ⚠️ No output for ${INACTIVITY_KILL_MS/1000}s — openclaw hung mid-task, killing`);
|
|
919
|
-
promiseSettled = true;
|
|
920
|
-
// cancelAgent kills the process tree; OpenClawCLI's close handler will reject runAgentTask
|
|
921
|
-
activeRunner.cancelAgent(agentId);
|
|
922
|
-
}
|
|
923
|
-
}, INACTIVITY_KILL_MS);
|
|
924
1009
|
};
|
|
925
1010
|
|
|
926
1011
|
// Track tool lifecycle
|
|
@@ -944,24 +1029,19 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
944
1029
|
// Wrap handlers to track activity
|
|
945
1030
|
const wrappedOutputHandler = activityWrapper(outputHandler);
|
|
946
1031
|
const wrappedToolHandler = activityWrapper(toolActivityHandler);
|
|
947
|
-
|
|
948
|
-
|
|
1032
|
+
|
|
949
1033
|
activeRunner.off('agent_output', outputHandler);
|
|
950
1034
|
activeRunner.off('tool_activity', toolActivityHandler);
|
|
951
|
-
activeRunner.off('agent_alive', aliveHandler);
|
|
952
1035
|
activeRunner.on('agent_output', wrappedOutputHandler);
|
|
953
1036
|
activeRunner.on('tool_activity', wrappedToolHandler);
|
|
954
|
-
activeRunner.on('agent_alive', wrappedAliveHandler);
|
|
955
1037
|
|
|
956
1038
|
// Capture cleanup as a callable closure so both try and catch paths can use it
|
|
957
1039
|
_cleanup = () => {
|
|
958
|
-
promiseSettled = true;
|
|
959
|
-
clearInactivityTimers();
|
|
1040
|
+
promiseSettled = true;
|
|
960
1041
|
activeRunner.off('agent_output', wrappedOutputHandler);
|
|
961
1042
|
activeRunner.off('agent_error', errorHandler);
|
|
962
1043
|
activeRunner.off('tool_activity', wrappedToolHandler);
|
|
963
1044
|
activeRunner.off('tool_activity', toolLifecycleHandler);
|
|
964
|
-
activeRunner.off('agent_alive', wrappedAliveHandler);
|
|
965
1045
|
activeRunner.off('agent_image', imageHandler);
|
|
966
1046
|
};
|
|
967
1047
|
|
|
@@ -981,13 +1061,73 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
981
1061
|
finalMessage = contextInfo;
|
|
982
1062
|
}
|
|
983
1063
|
|
|
1064
|
+
// If taskCwd is set, scan projects folder and auto-inject context for any project
|
|
1065
|
+
// mentioned by name in the user message — so the agent never needs to ask "where is it?"
|
|
1066
|
+
let projectContext = null;
|
|
1067
|
+
if (taskCwd && taskCwd !== agentWorkspaceDir) {
|
|
1068
|
+
try {
|
|
1069
|
+
const projects = readdirSync(taskCwd).filter(e => !e.startsWith('.'));
|
|
1070
|
+
// Find project dirs whose name appears in the user message (case-insensitive)
|
|
1071
|
+
const msg = userMessage.toLowerCase();
|
|
1072
|
+
const matched = projects.filter(p => msg.includes(p.toLowerCase()));
|
|
1073
|
+
if (matched.length > 0) {
|
|
1074
|
+
const snippets = [];
|
|
1075
|
+
for (const proj of matched.slice(0, 2)) {
|
|
1076
|
+
const projPath = path.join(taskCwd, proj);
|
|
1077
|
+
// Read package.json or first README for stack/description
|
|
1078
|
+
for (const fname of ['package.json', 'README.md', 'readme.md']) {
|
|
1079
|
+
const fpath = path.join(projPath, fname);
|
|
1080
|
+
if (existsSync(fpath)) {
|
|
1081
|
+
try {
|
|
1082
|
+
const raw = readFileSync(fpath, 'utf-8').slice(0, 800);
|
|
1083
|
+
snippets.push(`--- ${proj}/${fname} ---\n${raw}`);
|
|
1084
|
+
break;
|
|
1085
|
+
} catch { /* skip */ }
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
// If nested (e.g. "Faith Guide/Faith Guide App"), go one level deeper
|
|
1089
|
+
if (snippets.length === 0) {
|
|
1090
|
+
try {
|
|
1091
|
+
const sub = readdirSync(projPath).filter(e => !e.startsWith('.'));
|
|
1092
|
+
for (const s of sub.slice(0, 4)) {
|
|
1093
|
+
const subPath = path.join(projPath, s);
|
|
1094
|
+
// Try package.json first
|
|
1095
|
+
const pkgPath = path.join(subPath, 'package.json');
|
|
1096
|
+
if (existsSync(pkgPath)) {
|
|
1097
|
+
try { snippets.push(`--- ${proj}/${s}/package.json ---\n${readFileSync(pkgPath, 'utf-8').slice(0, 600)}`); break; } catch { /* skip */ }
|
|
1098
|
+
}
|
|
1099
|
+
// Detect Swift/iOS (Xcode project)
|
|
1100
|
+
const xcode = readdirSync(subPath).find(f => f.endsWith('.xcodeproj'));
|
|
1101
|
+
if (xcode) {
|
|
1102
|
+
// Read any .md file in this folder for context
|
|
1103
|
+
const md = readdirSync(subPath).find(f => f.endsWith('.md'));
|
|
1104
|
+
const mdContent = md ? readFileSync(path.join(subPath, md), 'utf-8').slice(0, 600) : '';
|
|
1105
|
+
const subContents = readdirSync(subPath).filter(e => !e.startsWith('.')).join(', ');
|
|
1106
|
+
snippets.push(`--- ${proj}/${s} --- (Swift/iOS Xcode project)\nContents: ${subContents}${mdContent ? '\n\n' + md + ':\n' + mdContent : ''}`);
|
|
1107
|
+
break;
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
if (snippets.length === 0) snippets.push(`--- ${proj}/ ---\nSubfolders: ${sub.join(', ')}`);
|
|
1111
|
+
} catch { /* skip */ }
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
if (snippets.length > 0) {
|
|
1115
|
+
projectContext = `[Project context pre-loaded — do NOT ask the user for this info:\n${snippets.join('\n\n')}\n]`;
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
} catch { /* non-fatal */ }
|
|
1119
|
+
}
|
|
1120
|
+
|
|
984
1121
|
// Inject platform context into EVERY message so the agent always knows:
|
|
985
1122
|
// 1. What platform it's running on and its URL
|
|
986
1123
|
// 2. Where the user's projects folder is
|
|
987
1124
|
// 3. Screenshot capabilities
|
|
988
1125
|
const platformContext = [
|
|
989
1126
|
`[System context:`,
|
|
990
|
-
`- Platform: AgentForge.ai. Dashboard: https://agentforgeai-production.up.railway.app/dashboard. CRITICAL: Always use the built-in 'browser' tool for ALL web browsing AND web searches — NEVER use the 'web_search' tool (no API keys are configured), NEVER run shell commands like 'open', 'google-chrome', 'chromium', or any OS command to launch a browser. The browser tool connects to AgentForge Browser (port 9223) automatically. To search: use browser to navigate to google.com
|
|
1127
|
+
`- Platform: AgentForge.ai. Dashboard: https://agentforgeai-production.up.railway.app/dashboard. CRITICAL: Always use the built-in 'browser' tool for ALL web browsing AND web searches — NEVER use the 'web_search' tool (no API keys are configured), NEVER run shell commands like 'open', 'google-chrome', 'chromium', or any OS command to launch a browser. The browser tool connects to AgentForge Browser (port 9223) automatically. To search: use browser to navigate to google.com.`,
|
|
1128
|
+
`- VIEWING/TESTING A WEB APP: Always check for a deployed URL first — look in the project for railway.toml, vercel.json, netlify.toml, .env, README.md, or package.json for a live URL. Open the deployed app in the browser. Only spin up a local server if there is genuinely no deployed version. Never default to localhost when a live URL might exist.`,
|
|
1129
|
+
`- LOCAL SERVERS: If you must use localhost, try http://127.0.0.1:PORT if http://localhost:PORT fails. Do not stop and ask — just try both.`,
|
|
1130
|
+
`- CREATING AGENTS AND CHATTING WITH THEM: Open the browser to https://agentforgeai-production.up.railway.app/dashboard. Click + to create a new agent. Click into that new agent's chat panel. Type a message into the chat input and send it — IN THE BROWSER, not via sessions_send. NEVER use sessions_send or sessions_spawn for this — those do not open a visible chat. The entire interaction happens inside the browser UI, start to finish, exactly like a human clicking around the dashboard.`,
|
|
991
1131
|
`- Your runner: ${useHampagent ? 'Hampagent' : 'OpenClaw'}.`,
|
|
992
1132
|
(!conversationHistory || conversationHistory.length === 0)
|
|
993
1133
|
? `- This is the first message. When greeting, say: "I'm [your name] — your ${useHampagent ? 'Hampagent' : 'OpenClaw'} agent running on AgentForge." Never say "autonomous AI agent". Never list capabilities in an intro.`
|
|
@@ -996,14 +1136,28 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
996
1136
|
? `- Your name is "${agentName}"${agentEmoji ? ` ${agentEmoji}` : ''}. This is your AgentForge identity. Do not ask the user who you are or what your name is — you already know.`
|
|
997
1137
|
: null,
|
|
998
1138
|
taskCwd && taskCwd !== agentWorkspaceDir
|
|
999
|
-
?
|
|
1139
|
+
? (() => {
|
|
1140
|
+
try {
|
|
1141
|
+
const entries = readdirSync(taskCwd).filter(e => !e.startsWith('.')).sort();
|
|
1142
|
+
return `- Projects folder: "${taskCwd}"\n Available projects: ${entries.join(', ')}\n To work on a project, go to "${taskCwd}/<project name>". Folder names may contain spaces — quote paths. Do NOT ask the user where code is or what stack it uses — the project list is above, find it and read the code.`;
|
|
1143
|
+
} catch {
|
|
1144
|
+
return `- Projects folder: "${taskCwd}". Find projects with ls. Do NOT ask the user where code is.`;
|
|
1145
|
+
}
|
|
1146
|
+
})()
|
|
1000
1147
|
: null,
|
|
1001
1148
|
agentWorkspaceDir
|
|
1002
1149
|
? `- Screenshots: screencapture -x ${agentWorkspaceDir}/ss1.png && sips -Z 1280 ${agentWorkspaceDir}/ss1.png (MUST resize — API rejects images over 5MB). Send to chat with: echo "AGENTFORGE_IMAGE:${agentWorkspaceDir}/ss1.png". Always screenshot visual work before saying done. NEVER use "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --headless for screenshots — use screencapture only.`
|
|
1003
1150
|
: `- Screenshots: screencapture -x /tmp/ss1.png && sips -Z 1280 /tmp/ss1.png (MUST resize — API rejects images over 5MB). Send to chat with: echo "AGENTFORGE_IMAGE:/tmp/ss1.png". Always screenshot visual work before saying done. NEVER use "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --headless for screenshots — use screencapture only.`,
|
|
1151
|
+
`- QUALITY STANDARD FOR VISUAL WORK: Screenshot everything and critique harshly. HARD FAILS: (1) any unreadable contrast. (2) NEVER default to dark background + purple/indigo/blue-gray accent + rounded cards — that specific combination is AI-generated slop. Dark themes are fine when intentional; the ban is on lazy defaults, not dark aesthetics. (3) NEVER use the rounded-square block logo with a centered symbol — teal/orange/blue square with a snowflake, asterisk, key, or geometric shape inside is the single most overused AI logo pattern and is immediately recognizable as LLM output. If the app needs a logo or icon, design something that actually reflects the product concept: wordmarks, custom letterforms, logotypes, or illustrated marks — not a colored square with a centered glyph. Every UI needs a deliberate distinct visual identity: a dominant color that fits the app, strong typographic scale, and one signature visual element that makes it look like a real product. Use actual Google Search for competitors and design research — use \`openclaw browser open "https://www.google.com/search?q=best+[app+type]+app"\` to open a VISIBLE browser tab the user can watch. Do NOT use any internal search command — it must open as a real browser tab. Once loaded: (1) screenshot the full results page including the AI overview at the top, (2) also open \`https://www.google.com/search?q=best+[app+type]+app&tbm=isch\` in another tab to see Google Images of real UI designs and screenshot that too, (3) keep the search tab open while you work — do not close it immediately, (4) then click through to the top-ranked modern products from the actual results. DO NOT pick sites from training data. DO NOT visit Wikipedia, old forums, or anything that is not a live modern product. Trust the search results on screen. NEVER seed or generate fake data in code — no generateSampleHistory(), no sampleData arrays, no initialWorkouts, no seed scripts, no "preloaded" content of any kind. When localStorage/DB is empty, show an empty state. "Realistic sample data pre-loaded" is not a feature, it is a bug. To test the app, open it in the browser and add data through the UI as a real user would.`,
|
|
1152
|
+
`- NARRATE YOUR WORK IN REAL TIME — THIS IS CRITICAL: The user is watching and sees nothing while you work. You must send short chat messages as you go so they know what is happening. Do NOT work silently for minutes then dump everything at the end. After every significant action, write a brief update: what you just did, what you found, what you are doing next. Examples: "Opening Obsidian and Roam to study the UI patterns..." / "Found it — Craft uses warm cream + serif type, very distinct. Stealing that direction." / "Building the graph view now, using D3 force-directed layout." / "Graph is working. Testing bi-directional links next." One or two sentences is enough — just keep the user in the loop continuously. Think of it like pair programming where you narrate out loud. Never go more than 60 seconds without sending an update.`,
|
|
1153
|
+
`- RESEARCH STANDARD: When asked to research before building, visit MINIMUM 5 sources. Navigate to actual live apps and take screenshots — reading a description is not research. Extract specific named UI patterns you are borrowing. Shallow research = shallow output.`,
|
|
1154
|
+
`- BROWSER PROFILE — ALWAYS VISIBLE: ALWAYS use profile="agentforge" on every single browser call: \`browser(action="...", profile="agentforge")\`. NEVER use the default headless openclaw browser — it is invisible to the user. The agentforge profile runs on port 9223 and is a real Chrome window the user can watch. If you omit profile="agentforge", the user cannot see what you are doing.`,
|
|
1155
|
+
`- BROWSER TAB RULE: \`openclaw browser navigate <url>\` loads the URL into the currently focused tab — which is the user's live AgentForge dashboard. Calling navigate with an external URL DESTROYS the user's chat view. Always use \`openclaw browser open <url>\` for any external site — it opens a new tab and leaves the dashboard untouched. When done researching, close those tabs with \`openclaw browser close <id>\` and restore the dashboard with \`openclaw browser focus <id>\`.`,
|
|
1156
|
+
`- CURRENT YEAR IS ${new Date().getFullYear()}. Never hardcode 2024 or 2025 in footers, copyright notices, or anywhere else. Always use ${new Date().getFullYear()}.`,
|
|
1157
|
+
`- BUILD QUALITY — NO LAZY ONE-FILE SETUPS: When asked to build something, build it properly. A real app has a real structure: separate files for server, frontend, styles, and logic. No single-file HTML dumps with everything jammed together. No "simple version" shortcuts. If the user asks for a web app, build a web app — with a backend (Node/Express or equivalent), a real frontend, a database or persistent storage, and proper file organization. The only exception is if the user explicitly asks for a quick prototype or single-file output. When in doubt, build the real thing.`,
|
|
1004
1158
|
`]`
|
|
1005
1159
|
].filter(Boolean).join('\n');
|
|
1006
|
-
finalMessage = platformContext + '\n\n' + finalMessage;
|
|
1160
|
+
finalMessage = platformContext + '\n\n' + (projectContext ? projectContext + '\n\n' : '') + finalMessage;
|
|
1007
1161
|
|
|
1008
1162
|
// If conversation history was loaded from DB (e.g. session expired, worker restarted,
|
|
1009
1163
|
// or user returning hours later), prepend it so the agent has full context.
|
|
@@ -1039,7 +1193,7 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
1039
1193
|
return text.replace(/\[System context:[\s\S]*?\n\]/g, '').trim();
|
|
1040
1194
|
};
|
|
1041
1195
|
const historyText = conversationHistory
|
|
1042
|
-
.slice(-
|
|
1196
|
+
.slice(-50) // last 50 messages of conversation history
|
|
1043
1197
|
.map(msg => {
|
|
1044
1198
|
const role = msg.role === 'user' ? 'User' : 'Assistant';
|
|
1045
1199
|
const content = msg.role === 'user'
|
|
@@ -1074,6 +1228,9 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
1074
1228
|
let taskResult;
|
|
1075
1229
|
let iterationMessage = finalMessage;
|
|
1076
1230
|
|
|
1231
|
+
// Agents use the browser concurrently — each tracks its own tab IDs.
|
|
1232
|
+
// No global lock; locking serialized all work behind any single browsing agent.
|
|
1233
|
+
|
|
1077
1234
|
while (iteration < MAX_ITERATIONS) {
|
|
1078
1235
|
iteration++;
|
|
1079
1236
|
|
|
@@ -1092,7 +1249,7 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
1092
1249
|
console.log(`[${taskId}] 🏃 Runner: ${useHampagent ? '⚡ HAMPAGENT' : '🔧 OPENCLAW'} — agent ${agentId} iteration ${iteration}`);
|
|
1093
1250
|
const runAgentStart = Date.now();
|
|
1094
1251
|
taskResult = await activeRunner.runAgentTask(
|
|
1095
|
-
agentId, iterationMessage, taskCwd, sessionId, iteration === 1 ? image : null, browserProfile, actualWorkDir
|
|
1252
|
+
agentId, iterationMessage, taskCwd, sessionId, iteration === 1 ? image : null, browserProfile, actualWorkDir, agentModel || null
|
|
1096
1253
|
);
|
|
1097
1254
|
const runAgentDuration = Date.now() - runAgentStart;
|
|
1098
1255
|
console.log(`[${taskId}] runAgentTask iteration ${iteration} returned after ${runAgentDuration}ms, success=${taskResult?.success}`);
|
|
@@ -1159,6 +1316,7 @@ export class AgentForgeWorker extends EventEmitter {
|
|
|
1159
1316
|
console.log(`[${taskId}] Got identity in ${Date.now() - identityStart}ms: ${identity.identityName}`);
|
|
1160
1317
|
}
|
|
1161
1318
|
|
|
1319
|
+
|
|
1162
1320
|
// Send completion with identity info, final response text, and sessionId for maestro
|
|
1163
1321
|
// Filter openclaw's "No reply from agent." placeholder — it appears when the agent only
|
|
1164
1322
|
// used tools with no text response (e.g. TTS-only tasks). If we send it, the browser's
|
|
@@ -1553,17 +1711,9 @@ Review and add specific steps, pitfalls, and patterns that helped succeed.
|
|
|
1553
1711
|
}
|
|
1554
1712
|
}
|
|
1555
1713
|
|
|
1556
|
-
// Record that an agent produced output (reset stuck detection)
|
|
1557
|
-
recordAgentActivity(agentId) {
|
|
1558
|
-
this.lastAgentActivity.set(agentId, Date.now());
|
|
1559
|
-
this.pingsSinceActivity.set(agentId, 0);
|
|
1560
|
-
}
|
|
1561
|
-
|
|
1562
1714
|
// Collect detailed diagnostics for debug agent
|
|
1563
1715
|
collectDiagnostics(agentId, taskId, error, reason) {
|
|
1564
1716
|
const taskInfo = this.runningTasks.get(taskId);
|
|
1565
|
-
const lastActivity = this.lastAgentActivity.get(agentId);
|
|
1566
|
-
const pings = this.pingsSinceActivity.get(agentId) || 0;
|
|
1567
1717
|
const processingTime = this.processingStartTime.get(agentId);
|
|
1568
1718
|
|
|
1569
1719
|
return {
|
|
@@ -1577,9 +1727,6 @@ Review and add specific steps, pitfalls, and patterns that helped succeed.
|
|
|
1577
1727
|
name: error.name
|
|
1578
1728
|
} : null,
|
|
1579
1729
|
activity: {
|
|
1580
|
-
lastActivityTime: lastActivity,
|
|
1581
|
-
timeSinceActivity: lastActivity ? Date.now() - lastActivity : null,
|
|
1582
|
-
pingsSinceActivity: pings,
|
|
1583
1730
|
processingStartTime: processingTime,
|
|
1584
1731
|
processingDuration: processingTime ? Date.now() - processingTime : null
|
|
1585
1732
|
},
|
|
@@ -1605,121 +1752,8 @@ Review and add specific steps, pitfalls, and patterns that helped succeed.
|
|
|
1605
1752
|
});
|
|
1606
1753
|
}
|
|
1607
1754
|
|
|
1608
|
-
// Check for stuck agents on each ping
|
|
1609
|
-
checkForStuckAgents() {
|
|
1610
|
-
for (const [agentId, isProcessing] of this.agentProcessing.entries()) {
|
|
1611
|
-
if (isProcessing) {
|
|
1612
|
-
// First, check if the process is still alive - if so, it's probably just thinking
|
|
1613
|
-
const agentInfo = this.cli.activeAgents?.get(agentId);
|
|
1614
|
-
const pid = agentInfo?.proc?.pid;
|
|
1615
|
-
if (pid) {
|
|
1616
|
-
try {
|
|
1617
|
-
// process.kill(pid, 0) checks if process exists without killing it
|
|
1618
|
-
process.kill(pid, 0);
|
|
1619
|
-
// Process is alive - record activity to prevent false stuck detection
|
|
1620
|
-
// This handles cases where the CLI is blocking on API calls with no stdout
|
|
1621
|
-
this.recordAgentActivity(agentId);
|
|
1622
|
-
} catch (e) {
|
|
1623
|
-
// Process is dead - let stuck detection proceed
|
|
1624
|
-
console.log(`⚠️ Agent ${agentId} process (PID ${pid}) appears dead`);
|
|
1625
|
-
}
|
|
1626
|
-
}
|
|
1627
|
-
|
|
1628
|
-
// Increment ping counter for this agent
|
|
1629
|
-
const pings = (this.pingsSinceActivity.get(agentId) || 0) + 1;
|
|
1630
|
-
this.pingsSinceActivity.set(agentId, pings);
|
|
1631
|
-
|
|
1632
|
-
// Check if there's an active task for this agent
|
|
1633
|
-
let hasActiveTask = false;
|
|
1634
|
-
for (const [taskId, taskInfo] of this.runningTasks.entries()) {
|
|
1635
|
-
if (taskInfo.agentId === agentId && !taskInfo.cancelled) {
|
|
1636
|
-
hasActiveTask = true;
|
|
1637
|
-
break;
|
|
1638
|
-
}
|
|
1639
|
-
}
|
|
1640
|
-
|
|
1641
|
-
// Use very long threshold if task is active (10 pings = 300s / 5 min)
|
|
1642
|
-
// OpenClaw embedded agents can spend 2-3+ minutes on complex reasoning
|
|
1643
|
-
// Only mark as stuck if truly unresponsive (no output for 5+ minutes)
|
|
1644
|
-
const threshold = hasActiveTask ? 10 : this.STUCK_PING_THRESHOLD;
|
|
1645
|
-
|
|
1646
|
-
// Log warning when agent is quiet but not yet stuck (helps with debugging)
|
|
1647
|
-
if (pings >= this.STUCK_PING_THRESHOLD && pings < threshold) {
|
|
1648
|
-
console.log(`⚠️ Agent ${agentId} quiet for ${pings} pings (${Math.round((Date.now() - this.lastAgentActivity.get(agentId)) / 1000)}s), but task is active - waiting...`);
|
|
1649
|
-
}
|
|
1650
|
-
|
|
1651
|
-
if (pings >= threshold) {
|
|
1652
|
-
const lastActivity = this.lastAgentActivity.get(agentId);
|
|
1653
|
-
const elapsed = lastActivity ? Math.round((Date.now() - lastActivity) / 1000) : '?';
|
|
1654
|
-
const reason = hasActiveTask ? 'no output for 300s+ AND process dead' : 'no active task';
|
|
1655
|
-
console.log(`🚨 STUCK DETECTED: Agent ${agentId} has had ${pings} pings with no activity (${reason}, last activity: ${elapsed}s ago)`);
|
|
1656
|
-
console.log(`🚨 Force resetting agent ${agentId} to accept new tasks`);
|
|
1657
|
-
|
|
1658
|
-
// Find the task for diagnostics
|
|
1659
|
-
let stuckTaskId = null;
|
|
1660
|
-
for (const [taskId, taskInfo] of this.runningTasks.entries()) {
|
|
1661
|
-
if (taskInfo.agentId === agentId && !taskInfo.cancelled) {
|
|
1662
|
-
stuckTaskId = taskId;
|
|
1663
|
-
break;
|
|
1664
|
-
}
|
|
1665
|
-
}
|
|
1666
|
-
|
|
1667
|
-
// Collect diagnostics before cleanup
|
|
1668
|
-
const diagnostics = this.collectDiagnostics(
|
|
1669
|
-
agentId,
|
|
1670
|
-
stuckTaskId,
|
|
1671
|
-
new Error(`Agent unresponsive for ${elapsed}s`),
|
|
1672
|
-
'stuck'
|
|
1673
|
-
);
|
|
1674
|
-
|
|
1675
|
-
// Force kill the process — try both runners
|
|
1676
|
-
this.cli.cancelAgent(agentId);
|
|
1677
|
-
this.hampagent?.cancelAgent(agentId);
|
|
1678
|
-
|
|
1679
|
-
// Clear all state for this agent
|
|
1680
|
-
this.agentProcessing.set(agentId, false);
|
|
1681
|
-
this.processingStartTime.delete(agentId);
|
|
1682
|
-
this.pingsSinceActivity.set(agentId, 0);
|
|
1683
|
-
|
|
1684
|
-
// Find and cancel any running task for this agent
|
|
1685
|
-
for (const [taskId, taskInfo] of this.runningTasks.entries()) {
|
|
1686
|
-
if (taskInfo.agentId === agentId && !taskInfo.cancelled) {
|
|
1687
|
-
taskInfo.cancelled = true;
|
|
1688
|
-
this.runningTasks.delete(taskId);
|
|
1689
|
-
this.send({
|
|
1690
|
-
type: 'task_failed',
|
|
1691
|
-
taskId,
|
|
1692
|
-
agentId,
|
|
1693
|
-
error: 'Agent became unresponsive (stuck detection triggered)'
|
|
1694
|
-
});
|
|
1695
|
-
}
|
|
1696
|
-
}
|
|
1697
|
-
|
|
1698
|
-
// Send debug report
|
|
1699
|
-
this.sendDebugReport(diagnostics, `Agent ${agentId} became unresponsive after ${elapsed}s with no activity`);
|
|
1700
|
-
|
|
1701
|
-
// Process any queued tasks
|
|
1702
|
-
const queue = this.agentQueues.get(agentId);
|
|
1703
|
-
if (queue && queue.length > 0) {
|
|
1704
|
-
console.log(`📤 Processing ${queue.length} queued tasks after stuck recovery`);
|
|
1705
|
-
setImmediate(() => this.processQueue(agentId));
|
|
1706
|
-
}
|
|
1707
|
-
}
|
|
1708
|
-
}
|
|
1709
|
-
}
|
|
1710
|
-
}
|
|
1711
|
-
|
|
1712
1755
|
async shutdown() {
|
|
1713
1756
|
console.log('🛑 Shutting down worker...');
|
|
1714
|
-
// Kill all active agent processes so they don't become orphans on restart
|
|
1715
|
-
if (this.cli && typeof this.cli.cancelAgent === 'function') {
|
|
1716
|
-
for (const agentId of this.agentProcessing.keys()) {
|
|
1717
|
-
if (this.agentProcessing.get(agentId)) {
|
|
1718
|
-
console.log(`🔪 Killing agent process: ${agentId}`);
|
|
1719
|
-
try { this.cli.cancelAgent(agentId); } catch (e) { /* already dead */ }
|
|
1720
|
-
}
|
|
1721
|
-
}
|
|
1722
|
-
}
|
|
1723
1757
|
if (this.ws) {
|
|
1724
1758
|
this.ws.close();
|
|
1725
1759
|
}
|