onbuzz 3.9.10 → 4.0.0

package/src/core/agentScheduler.js

@@ -29,6 +29,7 @@ import ContextInjectionService from '../services/contextInjectionService.js';
 import FlowContextService from '../services/flowContextService.js';
 import TokenCountingService from '../services/tokenCountingService.js';
 import ConversationCompactionService from '../services/conversationCompactionService.js';
+import { getToolSchemasForAgent } from '../tools/openaiFunctionSchemas.js';
 import {
   shouldAgentBeActive,
   getActiveAgents,
@@ -36,6 +37,16 @@ import {
 } from '../services/agentActivityService.js';
 
 class AgentScheduler {
+  /**
+   * Register the Discord service so the scheduler can ask whether a given
+   * agent is currently bridged — drives conditional `<external>` prompt
+   * injection per turn. Optional; unset if Discord isn't available.
+   */
+  setDiscordService(discordService) { this.discordService = discordService; }
+
+  /** Same as {@link setDiscordService} but for Telegram. */
+  setTelegramService(telegramService) { this.telegramService = telegramService; }
+
   constructor(agentPool, messageProcessor, aiService, logger, webSocketManager = null, modelRouterService = null, modelsService = null) {
     this.agentPool = agentPool;
     this.messageProcessor = messageProcessor;
@@ -93,11 +104,36 @@ class AgentScheduler {
     // Structure: Map<agentId, number> - count of consecutive messages without tools
     this.consecutiveNoToolMessages = new Map();
 
+    // Visualizer telemetry — see GET /scheduler endpoint.
+    // Ring buffer of recent cycle decisions + per-agent lock-acquire timestamps.
+    // Not load-bearing; safe to disable by never reading. See _recordCycle().
+    this._cycleHistory = [];         // newest last
+    this._cycleCounter = 0;
+    this._lockAcquiredAt = new Map(); // agentId -> Date when lock was acquired
+    this._CYCLE_HISTORY_MAX = 200;
+
+    // Empty-response stall tracker. Populated only when a cycle received an
+    // AI response whose content was empty/whitespace AND carried no tool calls.
+    // Reset on any productive cycle. Structure: Map<agentId, { count, firstAt }>.
+    this._emptyResponseTracker = new Map();
+
     // Configuration from constants (no magic numbers)
     this.iterationDelayMs = SCHEDULER_CONFIG.ITERATION_DELAY_MS;
     this.maxIterationsPerCycle = SCHEDULER_CONFIG.MAX_ITERATIONS_PER_CYCLE;
   }
 
+  /**
+   * Append one entry to the cycle-history ring buffer and trim to MAX.
+   * Telemetry for the scheduler visualizer.
+   * @private
+   */
+  _recordCycle(entry) {
+    this._cycleHistory.push(entry);
+    if (this._cycleHistory.length > this._CYCLE_HISTORY_MAX) {
+      this._cycleHistory.splice(0, this._cycleHistory.length - this._CYCLE_HISTORY_MAX);
+    }
+  }
+
   /**
    * Start the agent scheduler
    */
@@ -158,6 +194,9 @@ class AgentScheduler {
     this.stateHashHistory.clear();
     this.agentProcessingLocks.clear();
     this.consecutiveNoToolMessages.clear();
+    this._lockAcquiredAt.clear();
+    this._cycleHistory.length = 0;
+    this._emptyResponseTracker.clear();
 
     // Cleanup services
     if (this.tokenCountingService && this.tokenCountingService.cleanup) {
@@ -167,6 +206,85 @@ class AgentScheduler {
     this.logger.info('Agent Scheduler stopped');
   }
 
+  /**
+   * Atomic snapshot of scheduler + agent state, for the /scheduler visualizer.
+   *
+   * Everything here is read from in-memory state held by the scheduler or by
+   * AgentPool — there are no mutations, no network calls, and no awaits other
+   * than the single `agentPool.getAllAgents()` call (which is what the scheduler
+   * itself invokes every cycle, so calling it on demand is free).
+   *
+   * @returns {Promise<Object>}
+   */
+  async getState() {
+    const allAgents = await this.agentPool.getAllAgents();
+    const now = Date.now();
+    const iter = (allAgents && typeof allAgents[Symbol.iterator] === 'function' && !Array.isArray(allAgents))
+      ? allAgents.values()
+      : allAgents;
+
+    const agents = [];
+    for (const agent of iter) {
+      if (!agent) continue;
+      const activity = shouldAgentBeActive(agent);
+      const lockAcq = this._lockAcquiredAt.get(agent.id);
+      const tasks = agent.taskList?.tasks || [];
+      const nextActive = tasks.find(t => t.status === 'pending' || t.status === 'in_progress');
+      const queues = agent.messageQueues || {};
+      const activeAIReq = this.aiService?.getActiveRequest?.(agent.id) || null;
+
+      agents.push({
+        id: agent.id,
+        name: agent.name,
+        mode: agent.mode,
+        status: agent.status,
+        currentModel: agent.currentModel,
+        activity,
+        lockHeld: this.agentProcessingLocks.has(agent.id),
+        lockHeldMs: lockAcq ? now - lockAcq.getTime() : null,
+        delayEndTime: agent.delayEndTime || null,
+        delayedNow: !!(agent.delayEndTime && new Date(agent.delayEndTime).getTime() > now),
+        pausedUntil: agent.pausedUntil || null,
+        ttl: agent.ttl ?? null,
+        awaitingUserInput: agent.awaitingUserInput || null,
+        stopRequested: !!agent.stopRequested,
+        tasks: {
+          total: tasks.length,
+          pending:    tasks.filter(t => t.status === 'pending').length,
+          inProgress: tasks.filter(t => t.status === 'in_progress').length,
+          completed:  tasks.filter(t => t.status === 'completed').length,
+          nextPending: nextActive ? String(nextActive.content || '').slice(0, 120) : null,
+        },
+        queues: {
+          userMessages:       Array.isArray(queues.userMessages)       ? queues.userMessages.length       : 0,
+          interAgentMessages: Array.isArray(queues.interAgentMessages) ? queues.interAgentMessages.length : 0,
+          toolResults:        Array.isArray(queues.toolResults)        ? queues.toolResults.length        : 0,
+        },
+        activeAIRequest: activeAIReq,
+        stateHashHistory: (this.stateHashHistory.get(agent.id) || []).slice(-10),
+        emptyResponse: this._emptyResponseTracker.get(agent.id) || null,
+      });
+    }
+
+    return {
+      serverTime: new Date().toISOString(),
+      scheduler: {
+        running: this.isRunning,
+        iterationDelayMs: this.iterationDelayMs,
+        maxConcurrent: SCHEDULER_CONFIG.MAX_CONCURRENT_AGENTS || 3,
+        currentlyInFlight: this.agentProcessingLocks.size,
+        cycleCount: this._cycleCounter,
+        cycleHistoryMax: this._CYCLE_HISTORY_MAX,
+      },
+      locks: Array.from(this.agentProcessingLocks.keys()).map(id => ({
+        agentId: id,
+        heldMs: this._lockAcquiredAt.get(id) ? now - this._lockAcquiredAt.get(id).getTime() : null,
+      })),
+      cycles: this._cycleHistory.slice(),   // newest last; client reverses
+      agents,
+    };
+  }
+
   /**
    * Register an agent's session and ensure scheduler is running
    *
@@ -189,16 +307,26 @@ class AgentScheduler {
       });
     }
 
-    // Clear hash history when user sends a new message
-    // This provides a "fresh start" after user intervention, preventing false loop detection
-    if (context.triggeredBy === 'user-message') {
+    // Clear hash history when the user takes an action that should reset the
+    // loop detector: either sending a new message or flipping the agent back
+    // into AGENT mode. Without the mode-change reset, a loop intervention
+    // that partially failed (e.g. persistAgentState threw before the cleanup
+    // could run) could leave stale hashes that re-fire the loop detector on
+    // the first cycle after the user resumed autopilot.
+    const isFreshStart = context.triggeredBy === 'user-message'
+      || context.triggeredBy === 'mode-change-to-agent';
+
+    if (isFreshStart) {
       this.clearHashHistory(agentId);
-      this.logger.debug(`Hash history cleared for agent ${agentId} due to user message`);
+      this.logger.debug(`Hash history cleared for agent ${agentId}`, { triggeredBy: context.triggeredBy });
 
-      // Also reset consecutive no-tool counter on user message (fresh interaction)
+      // Also reset consecutive no-tool and empty-response trackers so the
+      // agent really does get a clean slate.
       if (this.consecutiveNoToolMessages.has(agentId)) {
         this.consecutiveNoToolMessages.set(agentId, 0);
-        this.logger.debug(`Consecutive no-tool counter reset for agent ${agentId} due to user message`);
+      }
+      if (this._emptyResponseTracker.has(agentId)) {
+        this._emptyResponseTracker.delete(agentId);
       }
     }
 
@@ -244,12 +372,18 @@ class AgentScheduler {
     if (this.agentProcessingLocks.has(agentId)) {
       this.agentProcessingLocks.delete(agentId);
     }
+    this._lockAcquiredAt.delete(agentId);
 
     // Clean up consecutive no-tool counter
     if (this.consecutiveNoToolMessages.has(agentId)) {
       this.consecutiveNoToolMessages.delete(agentId);
     }
 
+    // Clean up empty-response tracker
+    if (this._emptyResponseTracker.has(agentId)) {
+      this._emptyResponseTracker.delete(agentId);
+    }
+
     this.logger.debug(`Agent session cleaned up: ${agentId}`, { reason });
   }
 
@@ -326,6 +460,9 @@ class AgentScheduler {
           }
         });
       }
+      // Canonical state broadcast — ensures the UI flips the mode toggle
+      // even if it isn't listening for the legacy `execution_stopped` event.
+      await this.broadcastAgentStateUpdate(agentId, 'user-stopped');
 
       this.logger.info(`Agent execution stopped: ${agentId}`, {
         mode: agent.mode
@@ -365,27 +502,50 @@ class AgentScheduler {
     // Get all agents from pool and determine which should be active
     const allAgents = await this.agentPool.getAllAgents();
     const activeAgentResults = getActiveAgents(allAgents);
+    const totalAgents = (allAgents && typeof allAgents.size === 'number') ? allAgents.size : (Array.isArray(allAgents) ? allAgents.length : 0);
+    const currentlyInFlightAtStart = this.agentProcessingLocks.size;
+    const cycleN = ++this._cycleCounter;
+    const cycleAt = new Date().toISOString();
 
     if (activeAgentResults.length === 0) {
-      return; // No agents to process
+      // No agents want to run this cycle.
+      this._recordCycle({
+        n: cycleN, at: cycleAt, totalAgents,
+        activeCount: 0, active: [],
+        inFlightAtStart: currentlyInFlightAtStart,
+        skippedLocked: [], skippedConcurrency: [], launched: [],
+        outcome: 'idle',
+      });
+      return;
     }
 
     // Concurrency enforcement: count how many agents are already in-flight
     // from previous cycles, and only launch new ones up to the global cap.
     const maxConcurrent = SCHEDULER_CONFIG.MAX_CONCURRENT_AGENTS || 3;
-    const currentlyInFlight = this.agentProcessingLocks.size;
+    const currentlyInFlight = currentlyInFlightAtStart;
 
     // Filter out agents already being processed
+    const skippedLocked = [];
     const unlockedAgents = activeAgentResults.filter(r => {
       if (this.agentProcessingLocks.get(r.agentId)) {
         this.logger.debug(`Agent ${r.agentId} still processing from previous cycle, skipping`);
+        skippedLocked.push(r.agentId);
         return false;
       }
       return true;
     });
 
     if (unlockedAgents.length === 0) {
-      return; // All active agents already being processed
+      // Every active agent is wedged in a previous cycle's lock.
+      this._recordCycle({
+        n: cycleN, at: cycleAt, totalAgents,
+        activeCount: activeAgentResults.length,
+        active: activeAgentResults.map(r => ({ agentId: r.agentId, name: r.agent?.name, reason: r.reason, details: r.details })),
+        inFlightAtStart: currentlyInFlight,
+        skippedLocked, skippedConcurrency: [], launched: [],
+        outcome: 'all-locked',
+      });
+      return;
     }
 
     // Round-robin fairness: agents that launched in a recent cycle go to the back
@@ -399,9 +559,18 @@ class AgentScheduler {
     // Cap new launches so total in-flight never exceeds MAX_CONCURRENT_AGENTS
     const slotsAvailable = Math.max(0, maxConcurrent - currentlyInFlight);
     const agentsToLaunch = unlockedAgents.slice(0, slotsAvailable);
+    const skippedConcurrency = unlockedAgents.slice(slotsAvailable).map(r => r.agentId);
 
     if (agentsToLaunch.length === 0) {
       this.logger.debug(`Concurrency cap reached: ${currentlyInFlight}/${maxConcurrent} agents in-flight, ${unlockedAgents.length} waiting`);
+      this._recordCycle({
+        n: cycleN, at: cycleAt, totalAgents,
+        activeCount: activeAgentResults.length,
+        active: activeAgentResults.map(r => ({ agentId: r.agentId, name: r.agent?.name, reason: r.reason, details: r.details })),
+        inFlightAtStart: currentlyInFlight,
+        skippedLocked, skippedConcurrency, launched: [],
+        outcome: 'concurrency-cap',
+      });
       return;
     }
 
@@ -413,6 +582,18 @@ class AgentScheduler {
     // Track which agents we're launching for round-robin fairness
     this._lastLaunchedAgentIds = new Set(agentsToLaunch.map(r => r.agentId));
 
+    // Record the launch decision before firing so the visualizer sees it
+    // even if the async launch errors.
+    this._recordCycle({
+      n: cycleN, at: cycleAt, totalAgents,
+      activeCount: activeAgentResults.length,
+      active: activeAgentResults.map(r => ({ agentId: r.agentId, name: r.agent?.name, reason: r.reason, details: r.details })),
+      inFlightAtStart: currentlyInFlight,
+      skippedLocked, skippedConcurrency,
+      launched: agentsToLaunch.map(r => r.agentId),
+      outcome: 'launched',
+    });
+
     // Fire-and-forget: launch processing without awaiting.
     // Each agent is protected by its own lock in processAgent().
     // Next cycle (1s later) will pick up remaining agents when slots free up.
@@ -447,6 +628,7 @@ class AgentScheduler {
           });
           // Clean up on error - don't leave stale locks
           this.agentProcessingLocks.delete(agentId);
+          this._lockAcquiredAt.delete(agentId);
         }
       })
     );
@@ -475,6 +657,7 @@ class AgentScheduler {
 
     // Set processing lock (prevents concurrent processing of the same agent)
     this.agentProcessingLocks.set(agentId, true);
+    this._lockAcquiredAt.set(agentId, new Date());
 
     try {
       // Use centralized service to check if we should skip this iteration
@@ -531,14 +714,23 @@ class AgentScheduler {
         return;
       }
 
-      // Track whether we actually got a new AI response this cycle
-      let gotNewAIResponse = false;
+      // Track what happened this cycle. Two independent signals:
+      //   gotResponse — the AI service returned *anything* (vs null).
+      //   advanced    — the conversation actually moved forward (message
+      //                 appended or tools extracted). If false despite
+      //                 gotResponse=true, the model returned an empty or
+      //                 whitespace-only message with no tool calls — a
+      //                 "no-op cycle" that must NOT pollute stateHashHistory.
+      let gotResponse = false;
+      let advanced = false;
 
       // Process based on whether there are messages or agent needs autonomous processing
       if (totalMessages === 0 && agent.mode === AGENT_MODES.AGENT) {
         // AGENT mode with no messages - check for task-based work
         await this.autoCreateInitialTaskIfNeeded(agentId);
-        gotNewAIResponse = await this.processAgentAutonomously(agentId);
+        const result = await this.processAgentAutonomously(agentId);
+        gotResponse = result.gotResponse;
+        advanced = result.advanced;
       } else if (totalMessages > 0) {
         // Has messages to process
         const processedMessages = await this.processAgentQueues(agentId);
@@ -548,9 +740,8 @@ class AgentScheduler {
           const aiResponse = await this.getAgentAIResponse(agentId);
 
           if (aiResponse) {
-            // Process AI response and execute any tools
-            await this.processAIResponse(agentId, aiResponse);
-            gotNewAIResponse = true;
+            gotResponse = true;
+            advanced = await this.processAIResponse(agentId, aiResponse);
 
             // Clear token limit retry tracker on successful AI response
             this.clearTokenLimitRetryTracker(agentId);
@@ -561,12 +752,29 @@ class AgentScheduler {
       }
       // CHAT mode with no messages: do nothing - activity service will mark as inactive
 
-      // Only record state hash when we actually got a new AI response
-      // This prevents false loop detection when agent is idle
-      if (gotNewAIResponse) {
+      // Only record state hash when the conversation actually advanced. If
+      // the model returned an empty response (gotResponse=true, advanced=false),
+      // we're still in the same conversation state as before and recording
+      // would produce phantom duplicates that detectRepetitiveLoop misreads
+      // as progress toward a loop.
+      if (advanced) {
         this.recordStateHash(agentId, currentStateHash);
       }
 
+      // Empty-response stall detector. When the model returns an empty /
+      // whitespace-only response with no tool calls, we keep retrying — the
+      // next cycle may produce a real message. But we ring-buffer the empties
+      // so a genuinely stuck model eventually surfaces as a user-facing error
+      // instead of silently spinning forever.
+      if (gotResponse && !advanced) {
+        await this._trackEmptyResponse(agentId);
+      } else if (advanced) {
+        // Productive cycle — any previous stall has resolved itself.
+        if (this._emptyResponseTracker.has(agentId)) {
+          this._emptyResponseTracker.delete(agentId);
+        }
+      }
+
       // Decrement TTL (Time-to-Live) if set
       // TTL gives agents extra processing cycles after clearing tasks
       const agentForTtl = await this.agentPool.getAgent(agentId);
@@ -582,6 +790,7 @@ class AgentScheduler {
     } finally {
       // Always clear processing lock, even on errors
       this.agentProcessingLocks.delete(agentId);
+      this._lockAcquiredAt.delete(agentId);
     }
   }
 
@@ -683,6 +892,20 @@ class AgentScheduler {
       const toolIds = [...new Set(toolResults.map(m => m.toolId).filter(Boolean))];
       toolResultContent += `[Tool Results — ${toolCount} result${toolCount > 1 ? 's' : ''} from ${turnCount} tool batch${turnCount > 1 ? 'es' : ''}: ${toolIds.join(', ')}]\n`;
 
+      // Current-working-directory banner. Reasoning models regularly forget
+      // that `change-directory` persists the CWD across turns and then
+      // re-prepend the project folder name to relative paths, which nests
+      // directories unintentionally (".../foo/foo/..."). Surfacing the CWD
+      // at the top of every tool-results injection gives the model a stable
+      // anchor: "you are HERE, all relative paths resolve from HERE."
+      try {
+        const agentForCwd = await this.agentPool.getAgent(agentId);
+        const cwd = agentForCwd?.directoryAccess?.workingDirectory;
+        if (cwd) {
+          toolResultContent += `[CWD: ${cwd} — relative paths resolve from here; do NOT prepend the project folder name]\n`;
+        }
+      } catch { /* best-effort; never block the tool-results injection */ }
+
       if (turnCount === 1) {
         // Single batch — flat list (no sub-headers needed)
         toolResults.forEach(msg => {
@@ -805,11 +1028,11 @@ class AgentScheduler {
    */
   async addMessageToConversation(agentId, message, broadcast = true) {
     const agent = await this.agentPool.getAgent(agentId);
-    if (!agent) return;
+    if (!agent) return false;
 
     // Format message based on queue type
     let formattedMessage;
-    
+
     switch (message.queueType) {
       case 'toolResults': // Tool results
         formattedMessage = {
@@ -818,7 +1041,7 @@ class AgentScheduler {
           content: this.formatToolResult(message)
         };
         break;
-        
+
       case 'interAgentMessages': // Inter-agent messages
         formattedMessage = {
           ...message,
@@ -826,14 +1049,14 @@ class AgentScheduler {
           content: `Message from ${message.senderName || message.sender}: ${message.content}`
         };
         break;
-        
+
       case 'userMessages': // User messages
         formattedMessage = {
           ...message,
           role: MESSAGE_ROLES.USER
         };
         break;
-        
+
       default:
         formattedMessage = message;
     }
@@ -843,7 +1066,10 @@ class AgentScheduler {
       formattedMessage.timestamp = new Date().toISOString();
     }
 
-    // GUARD: Skip empty messages - they should never be added to history
+    // GUARD: Skip empty messages - they should never be added to history.
+    // Returning `false` lets the caller (processAIResponse) know the conversation
+    // did NOT advance, so it can avoid polluting stateHashHistory and instead
+    // mark the cycle as an "empty response" for the stall-detector.
     const messageContent = formattedMessage.content;
     if (!messageContent || (typeof messageContent === 'string' && !messageContent.trim())) {
       this.logger.warn(`Skipping empty message for agent ${agentId}`, {
@@ -851,7 +1077,7 @@ class AgentScheduler {
         queueType: message.queueType,
         hasContent: !!messageContent
       });
-      return; // Don't add empty messages
+      return false; // Don't add empty messages; caller should treat as "no progress"
     }
 
     // Add to conversation history
@@ -868,6 +1094,8 @@ class AgentScheduler {
     if (broadcast && this.shouldBroadcastMessage(formattedMessage)) {
       this.broadcastMessageUpdate(agentId, formattedMessage);
     }
+
+    return true;
   }
 
   /**
@@ -883,16 +1111,22 @@ class AgentScheduler {
     const aiResponse = await this.getAgentAIResponse(agentId);
 
     if (!aiResponse) {
-      return false; // No response - activity service will determine if we should continue
+      // No response - activity service will determine if we should continue.
+      // Reported as { gotResponse: false } so the caller doesn't count this
+      // as an "empty-response stall" (which is specifically about empty content).
+      return { gotResponse: false, advanced: false };
     }
 
-    // Process AI response and execute tools
-    await this.processAIResponse(agentId, aiResponse);
+    // Process AI response and execute tools. processAIResponse returns `true`
+    // when the conversation actually advanced (assistant message appended or
+    // tool calls extracted), and `false` when the response was empty/whitespace
+    // and carried no tools.
+    const advanced = await this.processAIResponse(agentId, aiResponse);
 
     // Clear token limit retry tracker on successful AI response
     this.clearTokenLimitRetryTracker(agentId);
 
-    return true; // We got a new AI response
+    return { gotResponse: true, advanced };
   }
 
   /**
@@ -1530,6 +1764,30 @@ class AgentScheduler {
         enhancedSystemPrompt = (enhancedSystemPrompt || '') + systemConstraints;
       }
 
+      // Inject external-channel routing guidance ONLY when the agent has
+      // at least one live bridge. Pulls the per-channel alias list from
+      // each service so the agent can address specific channels/threads
+      // via `<external to="alias">`. When the agent isn't bridged the
+      // paragraph is omitted — no point teaching routing for listeners
+      // that aren't there. Filter + alias matching logic live in
+      // services/channelFilter.js so this block stays declarative.
+      try {
+        const activeChannels = [];
+        if (typeof this.discordService?.getBridgedChannels === 'function') {
+          activeChannels.push(...this.discordService.getBridgedChannels(agentId));
+        }
+        if (typeof this.telegramService?.getBridgedChannels === 'function') {
+          activeChannels.push(...this.telegramService.getBridgedChannels(agentId));
+        }
+        if (activeChannels.length > 0) {
+          const { getExternalChannelPromptGuidance } = await import('../services/channelFilter.js');
+          const guidance = getExternalChannelPromptGuidance(activeChannels);
+          if (guidance) enhancedSystemPrompt = (enhancedSystemPrompt || '') + guidance;
+        }
+      } catch (err) {
+        this.logger.warn('[external-routing] failed to inject <external> guidance', { agentId, error: err.message });
+      }
+
       // Inject flow execution context if this is part of a flow
       try {
         const lastUserMsg = [...conversationHistory].reverse().find(m => m.role === 'user');
@@ -1564,6 +1822,15 @@ class AgentScheduler {
         ? userStreamingPref !== false
         : agent.streamingEnabled !== false; // Default to true
 
+      // Native function-call tools for the agent's enabled capabilities.
+      // Passed as `options.tools` to the backend. For Responses-API models
+      // (codex/o-series/gpt-5-pro) the backend forwards these to Azure so
+      // the model can emit native function_call events; the backend converts
+      // them back into the CLI's inline JSON-block format on the wire. For
+      // chat-completion models the backend ignores this field and the model
+      // uses the system prompt's inline-JSON instructions as before.
+      const toolSchemas = getToolSchemasForAgent(agent.capabilities || []);
+
       if (streamingEnabled && this.aiService.sendMessageStream) {
         // Build flow context if this is part of a flow execution
         const flowContext = lastUserMsg?.isFlowExecution ? {
@@ -1579,7 +1846,8 @@ class AgentScheduler {
           enhancedSystemPrompt,
           sessionId,
           agent.platformProvided,
-          flowContext
+          flowContext,
+          toolSchemas
         );
       }
 
@@ -1608,7 +1876,8 @@ class AgentScheduler {
             agentId: agentId,
             systemPrompt: enhancedSystemPrompt,
             sessionId: sessionId,
-            platformProvided: agent.platformProvided
+            platformProvided: agent.platformProvided,
+            tools: toolSchemas && toolSchemas.length > 0 ? toolSchemas : undefined,
           }
         );
 
@@ -1642,7 +1911,7 @@ class AgentScheduler {
    * @returns {Promise<Object>} Response object
    * @private
    */
-  async _getStreamingResponse(agentId, targetModel, messagesToSend, systemPrompt, sessionId, platformProvided, flowContext = null) {
+  async _getStreamingResponse(agentId, targetModel, messagesToSend, systemPrompt, sessionId, platformProvided, flowContext = null, tools = null) {
     // Generate a unique message ID for this streaming response
     const streamMessageId = `stream-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
 
@@ -1704,6 +1973,7 @@ class AgentScheduler {
           systemPrompt: systemPrompt,
           sessionId: sessionId,
           platformProvided: platformProvided,
+          tools: tools && tools.length > 0 ? tools : undefined,
           onChunk: (chunk) => {
             // First chunk — clear the prestream watchdog. Active streaming means
             // the model is responsive; we don't want to cut a live stream off.
@@ -1898,6 +2168,7 @@ class AgentScheduler {
       // Switch agent to CHAT mode - no delay, user can retry when ready
       agent.mode = AGENT_MODES.CHAT;
       await this.agentPool.persistAgentState(agentId);
+      await this.broadcastAgentStateUpdate(agentId, 'timeout');
 
       // Broadcast timeout notification to UI (toast + console log only, no chat message)
       if (this.webSocketManager && this.webSocketManager.broadcastToSession) {
@@ -1924,6 +2195,7 @@ class AgentScheduler {
 
       agent.delayEndTime = new Date(Date.now() + SCHEDULER_CONFIG.API_KEY_ERROR_DELAY_MS).toISOString();
       await this.agentPool.persistAgentState(agentId);
+      await this.broadcastAgentStateUpdate(agentId, 'api-key-error');
 
       // Add error message to agent's queue
       await this.agentPool.addToolResult(agentId, {
@@ -1939,6 +2211,7 @@ class AgentScheduler {
 
       agent.delayEndTime = new Date(Date.now() + SCHEDULER_CONFIG.RATE_LIMIT_DELAY_MS).toISOString();
       await this.agentPool.persistAgentState(agentId);
+      await this.broadcastAgentStateUpdate(agentId, 'rate-limit');
 
     } else if (errorMessage.includes('network') || errorMessage.includes('connection')) {
       // Network issues (non-timeout) - shorter delay and retry
@@ -1946,6 +2219,7 @@ class AgentScheduler {
 
       agent.delayEndTime = new Date(Date.now() + SCHEDULER_CONFIG.NETWORK_ERROR_DELAY_MS).toISOString();
       await this.agentPool.persistAgentState(agentId);
+      await this.broadcastAgentStateUpdate(agentId, 'network-error');
 
     } else if (this.isTokenLimitError(errorMessage)) {
       // Token/context limit error - trigger emergency compaction and retry
@@ -1958,6 +2232,7 @@ class AgentScheduler {
 
       agent.delayEndTime = new Date(Date.now() + SCHEDULER_CONFIG.UNKNOWN_ERROR_DELAY_MS).toISOString();
       await this.agentPool.persistAgentState(agentId);
+      await this.broadcastAgentStateUpdate(agentId, 'server-error');
 
       // Add error message to agent's queue
       await this.agentPool.addToolResult(agentId, {
@@ -2052,6 +2327,7 @@ class AgentScheduler {
       // Now show the error to the user
       agent.delayEndTime = new Date(Date.now() + SCHEDULER_CONFIG.UNKNOWN_ERROR_DELAY_MS).toISOString();
       await this.agentPool.persistAgentState(agentId);
+      await this.broadcastAgentStateUpdate(agentId, 'context-limit-exhausted');
 
       // Broadcast error to UI
       if (this.webSocketManager && this.webSocketManager.broadcastToSession) {
@@ -2324,9 +2600,18 @@ class AgentScheduler {
   }
 
   /**
-   * Process AI response and execute any tools
+   * Process AI response and execute any tools.
+   *
+   * Returns `true` when the conversation actually advanced (either the assistant
+   * message was appended or tool calls were queued for async execution), and
+   * `false` when the response had empty / whitespace content AND no tool calls.
+   * The caller (processAgent) uses this to decide whether to record the
+   * pre-cycle state hash — preventing the detectRepetitiveLoop pollution that
+   * empty responses would otherwise cause.
+   *
    * @param {string} agentId - Agent ID
    * @param {Object} aiResponse - AI service response
+   * @returns {Promise<boolean>} true if conversation advanced, false on dropped-empty
    * @private
    */
   async processAIResponse(agentId, aiResponse) {
@@ -2337,7 +2622,7 @@ class AgentScheduler {
     // Safety check: agent must exist
     if (!agent) {
       this.logger.warn(`Cannot process AI response - agent ${agentId} not found`);
-      return;
+      return false;
     }
 
     // Check if response contains tool calls
@@ -2377,13 +2662,32 @@ class AgentScheduler {
 
     // Normalize token usage field names (backend may send input_tokens/output_tokens
     // instead of prompt_tokens/completion_tokens depending on provider)
+    // NOTE: use ?? (not ||) so that a genuine 0 doesn't fall through to the next
+    // field — but also guard against the case where `tokenUsage` is entirely
+    // absent or empty. Emits a one-shot debug log so we can see the actual
+    // shape once per unusual response in server logs.
     let normalizedTokenUsage = null;
     if (aiResponse.tokenUsage) {
+      const u = aiResponse.tokenUsage;
       normalizedTokenUsage = {
-        prompt_tokens: aiResponse.tokenUsage.prompt_tokens || aiResponse.tokenUsage.input_tokens || 0,
-        completion_tokens: aiResponse.tokenUsage.completion_tokens || aiResponse.tokenUsage.output_tokens || 0,
-        total_tokens: aiResponse.tokenUsage.total_tokens || 0
+        prompt_tokens:     u.prompt_tokens     ?? u.input_tokens  ?? 0,
+        completion_tokens: u.completion_tokens ?? u.output_tokens ?? 0,
+        total_tokens:      u.total_tokens      ?? ((u.prompt_tokens ?? u.input_tokens ?? 0) + (u.completion_tokens ?? u.output_tokens ?? 0)),
       };
+      if (process.env.DEBUG_TOKEN_USAGE === '1' || normalizedTokenUsage.total_tokens === 0) {
+        this.logger.warn('[TOKEN-USAGE] non-zero expected but got zero or debug-requested', {
+          agentId,
+          model: aiResponse.model,
+          rawTokenUsage: u,
+          normalized: normalizedTokenUsage,
+        });
+      }
+    } else if (process.env.DEBUG_TOKEN_USAGE === '1') {
+      this.logger.warn('[TOKEN-USAGE] aiResponse has no tokenUsage field', {
+        agentId,
+        model: aiResponse.model,
+        aiResponseKeys: Object.keys(aiResponse || {}),
+      });
     }
 
     // Create response message
@@ -2400,18 +2704,27 @@ class AgentScheduler {
       pendingToolExecution: hasTools
     };
 
-    await this.addMessageToConversation(agentId, responseMessage, false);
+    const appended = await this.addMessageToConversation(agentId, responseMessage, false);
 
-    // IMMEDIATELY broadcast the AI response to UI (don't wait for tool execution)
-    if (this.shouldBroadcastMessage(responseMessage)) {
+    // IMMEDIATELY broadcast the AI response to UI (don't wait for tool execution).
+    // We only broadcast if the message actually made it into the conversation —
+    // broadcasting an empty message confuses the UI.
+    if (appended && this.shouldBroadcastMessage(responseMessage)) {
       const updatedAgent = await this.agentPool.getAgent(agentId);
       this.broadcastMessageUpdate(agentId, responseMessage, {
         agentCurrentModel: updatedAgent?.currentModel
       });
     }
 
-    // Execute tools ASYNCHRONOUSLY - don't block the response
+    // Execute tools ASYNCHRONOUSLY - don't block the response. Tools run even
+    // if the assistant message itself was empty and dropped, so long as the
+    // content string happened to contain tool-call markup.
     this._executeToolsAsync(agentId, aiResponse.content, sessionId, responseMessage.id);
+
+    // The cycle "made progress" if EITHER the assistant message was persisted
+    // or a tool call was extracted from the content (tools will surface new
+    // messages through the toolResults queue next cycle).
+    return appended || hasTools;
   }
 
   /**
@@ -2463,6 +2776,7 @@ class AgentScheduler {
         if (agent) {
           agent.delayEndTime = new Date(Date.now() + maxDelay).toISOString();
           await this.agentPool.persistAgentState(agentId);
+          await this.broadcastAgentStateUpdate(agentId, 'builtin-delay');
           this.logger.debug(`Agent ${agentId} - applying ${maxDelay}ms builtin delay for tool execution`);
         }
       }
@@ -2704,6 +3018,45 @@ class AgentScheduler {
    * @param {Object} agentInfo - Additional agent information for UI sync
    * @private
    */
+  /**
+   * Broadcast an agent state snapshot (mode, delayEndTime, awaitingUserInput,
+   * stopRequested) whenever the scheduler mutates one of those fields.
+   *
+   * The UI reducer merges this payload into its agent record, so a single
+   * helper call everywhere-the-state-changes keeps the UI in sync without
+   * adding one-off event types per scenario. `reason` is a short tag
+   * (`timeout`, `rate-limit`, `builtin-delay`, `loop-detected`, etc.) so the
+   * UI can render a human-readable cause if it wants to.
+   *
+   * Best-effort: a missing webSocketManager or a sessionless agent makes this
+   * a no-op. Never throws.
+   *
+   * @param {string} agentId
+   * @param {string} reason   Short tag describing WHY the state changed.
+   */
+  async broadcastAgentStateUpdate(agentId, reason = 'state-change') {
+    try {
+      if (!this.webSocketManager?.broadcastToSession) return;
+      const agent = await this.agentPool.getAgent(agentId);
+      if (!agent) return;
+      const sessionId = this.getAgentSession(agentId) || agent.sessionId || 'scheduler-session';
+      this.webSocketManager.broadcastToSession(sessionId, {
+        type: 'agent_state_updated',
+        data: {
+          agentId,
+          mode: agent.mode,
+          delayEndTime: agent.delayEndTime || null,
+          awaitingUserInput: agent.awaitingUserInput || null,
+          stopRequested: !!agent.stopRequested,
+          reason,
+          timestamp: new Date().toISOString(),
+        },
+      });
+    } catch (err) {
+      this.logger?.warn?.(`agent_state_updated broadcast failed: ${err.message}`, { agentId, reason });
+    }
+  }
+
   broadcastMessageUpdate(agentId, message, agentInfo = {}) {
     if (this.webSocketManager && this.webSocketManager.broadcastToSession) {
       // Get the session ID from session map, message, or fallback
@@ -3220,6 +3573,127 @@ class AgentScheduler {
     };
   }
 
+  /**
+   * Record a cycle where the model returned an empty or whitespace-only
+   * response that carried no tool calls. The scheduler keeps retrying — the
+   * next cycle may produce a real message — but we buffer the empties so
+   * we can surface a clear error to the user if the stall persists.
+   *
+   * @param {string} agentId
+   * @private
+   */
+  async _trackEmptyResponse(agentId) {
+    const now = Date.now();
+    const existing = this._emptyResponseTracker.get(agentId);
+    const entry = existing
+      ? { count: existing.count + 1, firstAt: existing.firstAt, lastAt: now }
+      : { count: 1, firstAt: now, lastAt: now };
+
+    this._emptyResponseTracker.set(agentId, entry);
+
+    this.logger.warn(`[EMPTY-RESPONSE] Agent ${agentId}: cycle produced no content`, {
+      count: entry.count,
+      elapsedMs: now - entry.firstAt,
+      threshold: SCHEDULER_CONFIG.EMPTY_RESPONSE_STALL_THRESHOLD,
+      windowMs: SCHEDULER_CONFIG.EMPTY_RESPONSE_STALL_WINDOW_MS,
+    });
+
+    const shouldStall =
+      entry.count >= SCHEDULER_CONFIG.EMPTY_RESPONSE_STALL_THRESHOLD &&
+      (now - entry.firstAt) >= SCHEDULER_CONFIG.EMPTY_RESPONSE_STALL_WINDOW_MS;
+
+    if (shouldStall) {
+      try {
+        await this._handleEmptyResponseStall(agentId, entry);
+      } finally {
+        this._emptyResponseTracker.delete(agentId);
+      }
+    }
+  }
+
+  /**
+   * Circuit breaker fired when an agent has produced N empty responses across
+   * at least the configured time window. Surfaces a user-facing error message,
+   * switches the agent to CHAT mode (scheduler will mark it inactive), and
+   * clears scheduler-side state that could otherwise keep re-firing.
+   *
+   * Mirrors the shape of handleRepetitiveLoop but with a different message
+   * explaining the actual failure mode.
+   *
+   * @param {string} agentId
+   * @param {{count: number, firstAt: number, lastAt: number}} entry
+   * @private
+   */
+  async _handleEmptyResponseStall(agentId, entry) {
+    const agent = await this.agentPool.getAgent(agentId);
+    if (!agent) return;
+
+    // Clear scheduler state FIRST so any throw in persistence / broadcast below
+    // can't leave us re-firing this handler next cycle.
+    this.stateHashHistory.delete(agentId);
+    this.consecutiveNoToolMessages.delete(agentId);
+
+    const elapsedSec = Math.round((entry.lastAt - entry.firstAt) / 1000);
+    const interventionMessage = `The model returned ${entry.count} empty responses in a row over ${elapsedSec}s. ` +
+      `I've switched to chat mode so you can retry or adjust. Common causes: ` +
+      `the selected model is unreachable, rate-limited, or rejecting the current ` +
+      `conversation. You can pick a different model or send a new message to try again.`;
+
+    const messageToAdd = {
+      id: `msg-empty-response-stall-${Date.now()}`,
+      role: 'assistant',
+      content: interventionMessage,
+      timestamp: new Date().toISOString(),
+      isSystemMessage: true,
+      emptyResponseStall: {
+        count: entry.count,
+        elapsedMs: entry.lastAt - entry.firstAt,
+      },
+    };
+
+    if (agent.conversations && agent.conversations.full) {
+      agent.conversations.full.messages.push(messageToAdd);
+      agent.conversations.full.lastUpdated = new Date().toISOString();
+    }
+
+    agent.mode = AGENT_MODES.CHAT;
+
+    // Persist + broadcast are best-effort — a failure here must NOT leave the
+    // agent in a re-firing state. Scheduler state was cleared above.
+    try {
+      await this.agentPool.persistAgentState(agentId);
+    } catch (err) {
+      this.logger.error(`Failed to persist agent after empty-response stall: ${err.message}`, { agentId });
+    }
+
+    const sessionId = this.getAgentSession(agentId) || agent.sessionId;
+    if (sessionId && this.webSocketManager && this.webSocketManager.broadcastToSession) {
+      try {
+        this.webSocketManager.broadcastToSession(sessionId, {
+          type: 'message_added',
+          data: { agentId, message: messageToAdd, type: 'empty_response_stall' },
+        });
+        this.webSocketManager.broadcastToSession(sessionId, {
+          type: 'agent_mode_changed',
+          data: {
+            agentId,
+            mode: AGENT_MODES.CHAT,
+            reason: 'empty_response_stall',
+            timestamp: new Date().toISOString(),
+          },
+        });
+      } catch (err) {
+        this.logger.warn(`Failed to broadcast empty-response stall: ${err.message}`, { agentId });
+      }
+    }
+    await this.broadcastAgentStateUpdate(agentId, 'empty-response-stall');
+
+    this.logger.error(`Agent ${agentId} switched to CHAT mode after ${entry.count} empty responses`, {
+      agentName: agent.name,
+      elapsedMs: entry.lastAt - entry.firstAt,
+    });
+  }
+
   /**
    * Record a state hash in the sliding window
    *
@@ -3260,6 +3734,14 @@ class AgentScheduler {
     const agent = await this.agentPool.getAgent(agentId);
     if (!agent) return;
 
+    // Clear scheduler state FIRST. If persistAgentState or a WebSocket broadcast
+    // throws below, we must NOT leave stale loop-detector state that would
+    // re-fire this handler on the next scheduler cycle — that was the exact
+    // "detects a loop repeatedly without adding a new message first" bug.
+    this.stateHashHistory.delete(agentId);
+    this._emptyResponseTracker.delete(agentId);
+    this.consecutiveNoToolMessages.delete(agentId);
+
     // Create a user-friendly message
     const interventionMessage = `I notice I've been producing similar responses repeatedly (${loopDetection.occurrences} times). This usually means I'm stuck or need your guidance. I've switched to chat mode so you can provide direction.
 
@@ -3289,35 +3771,43 @@ What would you like me to do next? You can:
 
     // Stop the agent execution - switch to chat mode
     agent.mode = AGENT_MODES.CHAT;
-    await this.agentPool.persistAgentState(agentId);
+
+    // Persist + broadcast are best-effort. Scheduler state was already cleared
+    // above, so a failure here can still leave the intervention un-persisted
+    // but won't cause the loop detector to re-fire.
+    try {
+      await this.agentPool.persistAgentState(agentId);
+    } catch (err) {
+      this.logger.error(`Failed to persist agent after loop detection: ${err.message}`, { agentId });
+    }
 
     // Broadcast the message to UI so it appears in the chat
     const sessionId = this.getAgentSession(agentId) || agent.sessionId;
     if (sessionId && this.webSocketManager && this.webSocketManager.broadcastToSession) {
-      // Broadcast the intervention message
-      this.webSocketManager.broadcastToSession(sessionId, {
-        type: 'message_added',
-        data: {
-          agentId,
-          message: messageToAdd,
-          type: 'loop_intervention'
-        }
-      });
+      try {
+        this.webSocketManager.broadcastToSession(sessionId, {
+          type: 'message_added',
+          data: {
+            agentId,
+            message: messageToAdd,
+            type: 'loop_intervention'
+          }
+        });
 
-      // Also broadcast mode change
-      this.webSocketManager.broadcastToSession(sessionId, {
-        type: 'agent_mode_changed',
-        data: {
-          agentId,
-          mode: AGENT_MODES.CHAT,
-          reason: 'loop_detected',
-          timestamp: new Date().toISOString()
-        }
-      });
+        this.webSocketManager.broadcastToSession(sessionId, {
+          type: 'agent_mode_changed',
+          data: {
+            agentId,
+            mode: AGENT_MODES.CHAT,
+            reason: 'loop_detected',
+            timestamp: new Date().toISOString()
+          }
+        });
+      } catch (err) {
+        this.logger.warn(`Failed to broadcast loop intervention: ${err.message}`, { agentId });
+      }
     }
-
-    // Clear the hash history for this agent (fresh start when user responds)
-    this.stateHashHistory.delete(agentId);
+    await this.broadcastAgentStateUpdate(agentId, 'loop-detected');
 
     this.logger.warn(`Agent ${agentId} stopped due to repetitive loop - awaiting user intervention`, {
       occurrences: loopDetection.occurrences,