crewly 1.2.3 → 1.2.6

package/dist/backend/backend/src/services/agent/runtime-exit-monitor.service.js

@@ -1,4 +1,5 @@
 import * as fs from 'fs/promises';
+import * as http from 'http';
 import { LoggerService } from '../core/logger.service.js';
 import { StorageService } from '../core/storage.service.js';
 import { getSessionBackendSync, createSessionCommandHelper, } from '../session/index.js';
@@ -9,7 +10,8 @@ import { getTerminalGateway } from '../../websocket/terminal.gateway.js';
 import { SHELL_PROMPT_PATTERNS } from '../continuation/patterns/idle-patterns.js';
 import { PtyActivityTrackerService } from './pty-activity-tracker.service.js';
 import { OrchestratorRestartService } from '../orchestrator/orchestrator-restart.service.js';
-import { CREWLY_CONSTANTS, ORCHESTRATOR_SESSION_NAME, ORCHESTRATOR_ROLE, RUNTIME_EXIT_CONSTANTS, AGENT_SUSPEND_CONSTANTS, SESSION_COMMAND_DELAYS, RUNTIME_TYPES, GEMINI_FAILURE_PATTERNS, GEMINI_FORCE_RESTART_PATTERNS, GEMINI_FAILURE_RETRY_CONSTANTS, } from '../../constants.js';
+import * as crypto from 'crypto';
+import { CREWLY_CONSTANTS, ORCHESTRATOR_SESSION_NAME, ORCHESTRATOR_ROLE, RUNTIME_EXIT_CONSTANTS, AGENT_SUSPEND_CONSTANTS, SESSION_COMMAND_DELAYS, RUNTIME_TYPES, GEMINI_FAILURE_PATTERNS, GEMINI_FORCE_RESTART_PATTERNS, CLAUDE_FATAL_PATTERNS, GEMINI_FAILURE_RETRY_CONSTANTS, GEMINI_READY_PATTERNS, WEB_CONSTANTS, AGENT_HEARTBEAT_MONITOR_CONSTANTS, } from '../../constants.js';
 import { delay } from '../../utils/async.utils.js';
 /**
  * Service that monitors PTY sessions for runtime exit patterns.
@@ -28,6 +30,10 @@ import { delay } from '../../utils/async.utils.js';
  * needs recovery. These failure patterns bypass the shell prompt verification
  * since the CLI may still be running but non-functional.
  *
+ * For Claude Code, the service detects fatal API errors (e.g. thinking
+ * block corruption) that make the CLI permanently non-functional within
+ * the current session. These trigger immediate recovery without retry.
+ *
  * @example
  * ```typescript
  * const monitor = RuntimeExitMonitorService.getInstance();
@@ -43,6 +49,14 @@ export class RuntimeExitMonitorService {
     onExitDetectedCallback;
     agentRegistrationService = null;
     taskTrackingService = null;
+    eventBusService = null;
+    /**
+     * Restart timestamps per session, persisted across monitoring cycles.
+     * Unlike MonitoredSession (which is recreated on each startMonitoring call),
+     * this map survives agent restarts to prevent infinite restart loops
+     * when agents keep crashing (e.g. during network outages).
+     */
+    restartHistory = new Map();
     constructor() {
         this.logger = LoggerService.getInstance().createComponentLogger('RuntimeExitMonitorService');
     }
@@ -89,6 +103,14 @@ export class RuntimeExitMonitorService {
     setTaskTrackingService(service) {
         this.taskTrackingService = service;
     }
+    /**
+     * Set the EventBusService dependency for publishing agent lifecycle events.
+     *
+     * @param service - The EventBusService instance
+     */
+    setEventBusService(service) {
+        this.eventBusService = service;
+    }
     /**
      * Start monitoring a PTY session for runtime exit patterns.
      *
@@ -163,6 +185,10 @@ export class RuntimeExitMonitorService {
     /**
      * Stop monitoring a PTY session.
      *
+     * Also prunes the restartHistory entry for this session if it has no
+     * recent timestamps within the cooldown window, preventing slow memory
+     * leak for terminated sessions.
+     *
      * @param sessionName - PTY session name
      */
     stopMonitoring(sessionName) {
@@ -178,6 +204,21 @@ export class RuntimeExitMonitorService {
         }
         monitored.unsubscribe();
         this.sessions.delete(sessionName);
+        // Fix 3: Clean up restartHistory for terminated sessions to prevent
+        // unbounded Map growth. Keep the entry only if it has recent timestamps
+        // within the cooldown window (they may still be needed if the session
+        // is restarted soon). Delete the entry entirely otherwise.
+        const timestamps = this.restartHistory.get(sessionName);
+        if (timestamps) {
+            const windowStart = Date.now() - AGENT_HEARTBEAT_MONITOR_CONSTANTS.COOLDOWN_WINDOW_MS;
+            const recent = timestamps.filter(ts => ts > windowStart);
+            if (recent.length === 0) {
+                this.restartHistory.delete(sessionName);
+            }
+            else {
+                this.restartHistory.set(sessionName, recent);
+            }
+        }
         this.logger.debug('Stopped runtime exit monitoring', { sessionName });
     }
     /**
@@ -194,6 +235,7 @@ export class RuntimeExitMonitorService {
         for (const sessionName of sessionNames) {
             this.stopMonitoring(sessionName);
         }
+        this.restartHistory.clear();
         this.logger.debug('All runtime exit monitors destroyed');
     }
     /**
@@ -229,11 +271,19 @@ export class RuntimeExitMonitorService {
     /**
      * Confirm exit by checking for a shell prompt, then react.
      *
+     * Runtime-specific failure checks are gated by runtimeType so that
+     * patterns for one runtime cannot accidentally match another (e.g. a
+     * Codex exit matching Claude or Gemini patterns).
+     *
      * For Gemini CLI failure patterns (API errors, quota exhaustion, etc.),
      * the system retries with exponential backoff before declaring the agent
      * dead. Gemini CLI often recovers automatically from transient API errors
      * (RESOURCE_EXHAUSTED, UNAVAILABLE, Connection error). Only after
      * MAX_RETRIES does the system proceed with the exit/restart flow.
+     *
+     * For Claude Code fatal patterns (thinking block corruption, etc.),
+     * the system skips retry and forces immediate recovery since these
+     * errors are permanently unrecoverable within the same session.
      */
     async confirmAndReact(sessionName, helper) {
         const monitored = this.sessions.get(sessionName);
@@ -245,46 +295,11 @@ export class RuntimeExitMonitorService {
             // Verify shell prompt is visible (avoids false positives)
             const shellPromptConfirmed = this.verifyExitWithShellPrompt(sessionName, helper);
             if (!shellPromptConfirmed) {
-                // Codex: "Conversation interrupted" indicates the run was aborted and
-                // often leaves the agent stranded without completion callbacks. Treat it
-                // as an actionable runtime failure even if shell prompt isn't visible.
-                const isCodexInterrupted = monitored.runtimeType === RUNTIME_TYPES.CODEX_CLI
-                    && /Conversation interrupted/i.test(monitored.buffer);
-                if (isCodexInterrupted) {
-                    this.logger.warn('Codex interruption detected, proceeding with recovery flow', {
-                        sessionName,
-                    });
-                    // fall through to exit/restart handling below
-                }
-                // For Gemini CLI: detect either retryable failure patterns or forced
-                // restart/update markers that require immediate recovery handling.
-                const isGeminiFailure = monitored.runtimeType === RUNTIME_TYPES.GEMINI_CLI
-                    && GEMINI_FAILURE_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
-                const isGeminiForceRestart = monitored.runtimeType === RUNTIME_TYPES.GEMINI_CLI
-                    && GEMINI_FORCE_RESTART_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
-                if (!isGeminiFailure && !isGeminiForceRestart && !isCodexInterrupted) {
+                const runtimeSpecificMatch = await this.checkRuntimeSpecificFailure(sessionName, monitored, helper);
+                if (!runtimeSpecificMatch) {
                     this.logger.debug('Exit pattern matched but shell prompt not confirmed, ignoring', { sessionName });
                     return;
                 }
-                if (isGeminiFailure || isGeminiForceRestart) {
-                    // Some Gemini failures are transient; others (auto-update/restart)
-                    // should trigger immediate recovery and task re-delivery.
-                    const forceRestart = isGeminiForceRestart;
-                    if (forceRestart) {
-                        this.logger.warn('Gemini auto-update interruption detected, forcing recovery flow', {
-                            sessionName,
-                        });
-                    }
-                    else {
-                        // Retry with exponential backoff before declaring exit.
-                        // Gemini CLI can recover from transient API errors automatically.
-                        const retryResult = await this.handleGeminiFailureWithRetry(sessionName, monitored, helper);
-                        if (retryResult === 'recovered') {
-                            return;
-                        }
-                        // retryResult === 'exhausted' — fall through to exit flow
-                    }
-                }
             }
             // Mark as detected to prevent double-processing
             monitored.exitDetected = true;
@@ -294,179 +309,295 @@ export class RuntimeExitMonitorService {
                 role: monitored.role,
             });
             // Fire the exit-detected callback (used to cancel pending registrations)
-            if (this.onExitDetectedCallback) {
-                try {
-                    this.onExitDetectedCallback(sessionName);
+            this.fireExitDetectedCallback(sessionName);
+            // Try agent restart if it has in-progress tasks (non-orchestrator only)
+            if (monitored.role !== ORCHESTRATOR_ROLE && this.agentRegistrationService && this.taskTrackingService) {
+                const restarted = await this.tryAgentRestartWithTasks(sessionName, monitored);
+                if (restarted)
+                    return;
+            }
+            // Orchestrator-specific restart
+            if (sessionName === ORCHESTRATOR_SESSION_NAME) {
+                const restarted = await this.tryOrchestratorRestart(sessionName, monitored);
+                if (restarted)
+                    return;
+            }
+            // Normal inactive flow: update status, capture memory, broadcast
+            await this.transitionToInactive(sessionName, monitored);
+            // Cleanup this subscription
+            this.stopMonitoring(sessionName);
+        }
+        finally {
+            monitored.confirmationInFlight = false;
+        }
+    }
+    /**
+     * Check for runtime-specific failure patterns when shell prompt is not confirmed.
+     *
+     * Gated by runtimeType so patterns for one runtime cannot accidentally
+     * match another. Returns true if a runtime-specific failure was detected,
+     * or 'deferred' handling occurred (Gemini retry).
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state
+     * @param helper - Session command helper
+     * @returns True if a runtime-specific failure was confirmed
+     */
+    async checkRuntimeSpecificFailure(sessionName, monitored, helper) {
+        if (monitored.runtimeType === RUNTIME_TYPES.CODEX_CLI) {
+            const isCodexInterrupted = /Conversation interrupted/i.test(monitored.buffer);
+            if (isCodexInterrupted) {
+                this.logger.warn('Codex interruption detected, proceeding with recovery flow', { sessionName });
+                return true;
+            }
+        }
+        else if (monitored.runtimeType === RUNTIME_TYPES.CLAUDE_CODE) {
+            const isClaudeFatal = CLAUDE_FATAL_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
+            if (isClaudeFatal) {
+                const detectedPattern = CLAUDE_FATAL_PATTERNS.find((p) => p.test(monitored.buffer))?.source;
+                this.logger.warn('Claude Code fatal error detected, forcing immediate recovery', {
+                    sessionName,
+                    detectedPattern,
+                });
+                return true;
+            }
+        }
+        else if (monitored.runtimeType === RUNTIME_TYPES.GEMINI_CLI) {
+            const isGeminiFailure = GEMINI_FAILURE_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
+            const isGeminiForceRestart = GEMINI_FORCE_RESTART_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
+            if (isGeminiFailure || isGeminiForceRestart) {
+                if (isGeminiForceRestart) {
+                    this.logger.warn('Gemini auto-update interruption detected, forcing recovery flow', { sessionName });
                 }
-                catch (error) {
-                    this.logger.warn('onExitDetected callback error', {
-                        sessionName,
-                        error: error instanceof Error ? error.message : String(error),
-                    });
+                else {
+                    const retryResult = await this.handleGeminiFailureWithRetry(sessionName, monitored, helper);
+                    if (retryResult === 'recovered' || retryResult === 'deferred') {
+                        return false; // Not a confirmed exit — caller should return early
+                    }
                 }
+                return true;
             }
-            // Check for in-progress tasks — restart agent if any exist (non-orchestrator only)
-            if (monitored.role !== ORCHESTRATOR_ROLE && this.agentRegistrationService && this.taskTrackingService) {
+        }
+        return false;
+    }
+    /**
+     * Fire the onExitDetected callback safely.
+     *
+     * @param sessionName - PTY session name
+     */
+    fireExitDetectedCallback(sessionName) {
+        if (!this.onExitDetectedCallback)
+            return;
+        try {
+            this.onExitDetectedCallback(sessionName);
+        }
+        catch (error) {
+            this.logger.warn('onExitDetected callback error', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    /**
+     * Try to restart an agent that has in-progress tasks.
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state
+     * @returns True if the agent was successfully restarted (caller should return)
+     */
+    async tryAgentRestartWithTasks(sessionName, monitored) {
+        try {
+            const tasks = await this.taskTrackingService.getTasksForTeamMember(monitored.memberId || '');
+            const activeTasks = tasks.filter(t => t.status === 'assigned' || t.status === 'active' || t.status === 'pending_assignment');
+            if (activeTasks.length === 0)
+                return false;
+            if (!this.isAgentRestartAllowed(sessionName)) {
+                this.logger.warn('Agent restart cooldown active, skipping restart', {
+                    sessionName,
+                    activeTaskCount: activeTasks.length,
+                    maxRestartsPerWindow: AGENT_HEARTBEAT_MONITOR_CONSTANTS.MAX_RESTARTS_PER_WINDOW,
+                    cooldownWindowMs: AGENT_HEARTBEAT_MONITOR_CONSTANTS.COOLDOWN_WINDOW_MS,
+                });
+                this.notifyOrchestratorOfFailure(sessionName, 'runtime_exited_cooldown', activeTasks, false);
+                return false;
+            }
+            this.logger.info('Runtime exit with in-progress tasks, attempting restart', {
+                sessionName,
+                activeTaskCount: activeTasks.length,
+            });
+            try {
+                await this.restartAgentWithTasks(sessionName, monitored, activeTasks);
+                this.recordAgentRestart(sessionName);
+                this.notifyOrchestratorOfFailure(sessionName, 'runtime_exited', activeTasks, true);
+                this.stopMonitoring(sessionName);
+                return true;
+            }
+            catch (restartError) {
+                this.logger.warn('Agent restart after runtime exit failed, falling back to inactive', {
+                    sessionName,
+                    error: restartError instanceof Error ? restartError.message : String(restartError),
+                });
+                this.recordAgentRestart(sessionName);
+                this.notifyOrchestratorOfFailure(sessionName, 'runtime_exited', activeTasks, false);
+                return false;
+            }
+        }
+        catch (error) {
+            this.logger.warn('Failed to check in-progress tasks after runtime exit', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return false;
+        }
+    }
+    /**
+     * Try to restart the orchestrator after runtime exit.
+     *
+     * Sets status to inactive before restart to prevent queue processor
+     * from delivering messages during postInitialize, captures session
+     * memory, and broadcasts restart status.
+     *
+     * @param sessionName - PTY session name (should be ORCHESTRATOR_SESSION_NAME)
+     * @param monitored - Monitored session state
+     * @returns True if orchestrator was successfully restarted (caller should return)
+     */
+    async tryOrchestratorRestart(sessionName, monitored) {
+        // Set status to inactive BEFORE restart so the queue processor
+        // doesn't deliver messages during postInitialize.
+        try {
+            const storageService = StorageService.getInstance();
+            await storageService.updateAgentStatus(sessionName, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE);
+        }
+        catch {
+            // Best-effort — continue with restart
+        }
+        // Capture session memory before restart
+        try {
+            const sessionMemoryService = SessionMemoryService.getInstance();
+            await sessionMemoryService.onSessionEnd(sessionName, monitored.role, process.cwd());
+        }
+        catch (error) {
+            this.logger.warn('Failed to capture session memory before orchestrator restart', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+        try {
+            const restartService = OrchestratorRestartService.getInstance();
+            const success = await restartService.attemptRestart();
+            if (success) {
+                this.logger.info('Orchestrator restarted after runtime exit', { sessionName });
+                // Broadcast STARTED (not ACTIVE) — agent becomes ACTIVE only
+                // after registration prompt is processed and MCP call completes.
                 try {
-                    const tasks = await this.taskTrackingService.getTasksForTeamMember(monitored.memberId || '');
-                    const activeTasks = tasks.filter(t => t.status === 'assigned' || t.status === 'active' || t.status === 'pending_assignment');
-                    if (activeTasks.length > 0) {
-                        this.logger.info('Runtime exit with in-progress tasks, attempting restart', {
+                    const terminalGateway = getTerminalGateway();
+                    if (terminalGateway) {
+                        terminalGateway.broadcastOrchestratorStatus({
                             sessionName,
-                            activeTaskCount: activeTasks.length,
+                            agentStatus: CREWLY_CONSTANTS.AGENT_STATUSES.STARTED,
+                            reason: 'runtime_exit_restart',
                         });
-                        try {
-                            await this.restartAgentWithTasks(sessionName, monitored, activeTasks);
-                            // Cleanup this subscription (restart succeeded, skip inactive flow)
-                            this.stopMonitoring(sessionName);
-                            return;
-                        }
-                        catch (restartError) {
-                            this.logger.warn('Agent restart after runtime exit failed, falling back to inactive', {
-                                sessionName,
-                                error: restartError instanceof Error ? restartError.message : String(restartError),
-                            });
-                            // Fall through to normal inactive flow
-                        }
                     }
                 }
-                catch (error) {
-                    this.logger.warn('Failed to check in-progress tasks after runtime exit', {
-                        sessionName,
-                        error: error instanceof Error ? error.message : String(error),
-                    });
-                    // Fall through to normal inactive flow
-                }
-            }
-            // Orchestrator-specific restart: attempt auto-restart when runtime exits
-            if (sessionName === ORCHESTRATOR_SESSION_NAME) {
-                // CRITICAL: Set status to inactive BEFORE restart so the queue
-                // processor doesn't deliver messages during postInitialize.
-                // Without this, the old 'active' status persists in storage and
-                // the queue processor sends chat messages that get concatenated
-                // into /directory add commands in the new Gemini CLI session.
-                try {
-                    const storageService = StorageService.getInstance();
-                    await storageService.updateAgentStatus(sessionName, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE);
-                }
                 catch {
-                    // Best-effort — continue with restart
-                }
-                // Capture session memory before restart
-                try {
-                    const sessionMemoryService = SessionMemoryService.getInstance();
-                    await sessionMemoryService.onSessionEnd(sessionName, monitored.role, process.cwd());
-                }
-                catch (error) {
-                    this.logger.warn('Failed to capture session memory before orchestrator restart', {
-                        sessionName,
-                        error: error instanceof Error ? error.message : String(error),
-                    });
+                    // Best-effort broadcast
                 }
+                this.stopMonitoring(sessionName);
+                return true;
+            }
+            this.logger.warn('Orchestrator restart after runtime exit failed or not allowed, falling back to inactive', {
+                sessionName,
+            });
+        }
+        catch (error) {
+            this.logger.warn('Orchestrator restart attempt threw error, falling back to inactive', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+        return false;
+    }
+    /**
+     * Transition agent to inactive state: update storage, capture memory,
+     * publish events, and broadcast WebSocket status.
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state
+     */
+    async transitionToInactive(sessionName, monitored) {
+        // Update agent status to inactive
+        try {
+            const storageService = StorageService.getInstance();
+            await storageService.updateAgentStatus(sessionName, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE);
+            this.logger.info('Agent status updated to inactive after runtime exit', { sessionName });
+            // Publish agent:inactive event to EventBus so orchestrator is notified
+            if (this.eventBusService) {
                 try {
-                    const restartService = OrchestratorRestartService.getInstance();
-                    const success = await restartService.attemptRestart();
-                    if (success) {
-                        this.logger.info('Orchestrator restarted after runtime exit', { sessionName });
-                        // Broadcast restarted status as STARTED (not ACTIVE).
-                        // The agent becomes ACTIVE only after the registration prompt
-                        // is fully processed and the MCP registration call completes.
-                        // Broadcasting ACTIVE prematurely causes the queue processor
-                        // to deliver messages during postInitialize.
-                        try {
-                            const terminalGateway = getTerminalGateway();
-                            if (terminalGateway) {
-                                terminalGateway.broadcastOrchestratorStatus({
-                                    sessionName,
-                                    agentStatus: CREWLY_CONSTANTS.AGENT_STATUSES.STARTED,
-                                    reason: 'runtime_exit_restart',
-                                });
-                            }
-                        }
-                        catch {
-                            // Best-effort broadcast
-                        }
-                        this.stopMonitoring(sessionName);
-                        return;
-                    }
-                    this.logger.warn('Orchestrator restart after runtime exit failed or not allowed, falling back to inactive', {
-                        sessionName,
-                    });
+                    await this.publishInactiveEvent(sessionName, monitored, storageService);
                 }
-                catch (error) {
-                    this.logger.warn('Orchestrator restart attempt threw error, falling back to inactive', {
+                catch (eventError) {
+                    this.logger.warn('Failed to publish agent:inactive event to EventBus', {
                         sessionName,
-                        error: error instanceof Error ? error.message : String(error),
+                        error: eventError instanceof Error ? eventError.message : String(eventError),
                     });
                 }
-                // Fall through to normal inactive flow
             }
-            // Update agent status to inactive
+        }
+        catch (error) {
+            this.logger.warn('Failed to update agent status after runtime exit', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+        // Capture session memory (skip for orchestrator — already captured above)
+        if (sessionName !== ORCHESTRATOR_SESSION_NAME) {
             try {
-                const storageService = StorageService.getInstance();
-                await storageService.updateAgentStatus(sessionName, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE);
-                this.logger.info('Agent status updated to inactive after runtime exit', { sessionName });
+                const sessionMemoryService = SessionMemoryService.getInstance();
+                await sessionMemoryService.onSessionEnd(sessionName, monitored.role, process.cwd());
             }
             catch (error) {
-                this.logger.warn('Failed to update agent status after runtime exit', {
+                this.logger.warn('Failed to capture session memory after runtime exit', {
                     sessionName,
                     error: error instanceof Error ? error.message : String(error),
                 });
             }
-            // Capture session memory (skip for orchestrator — already captured above)
-            if (sessionName !== ORCHESTRATOR_SESSION_NAME) {
-                try {
-                    const sessionMemoryService = SessionMemoryService.getInstance();
-                    await sessionMemoryService.onSessionEnd(sessionName, monitored.role, process.cwd());
-                }
-                catch (error) {
-                    this.logger.warn('Failed to capture session memory after runtime exit', {
-                        sessionName,
-                        error: error instanceof Error ? error.message : String(error),
-                    });
+        }
+        // Broadcast WebSocket event
+        try {
+            const terminalGateway = getTerminalGateway();
+            if (terminalGateway) {
+                const statusPayload = {
+                    sessionName,
+                    agentStatus: CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE,
+                    reason: 'runtime_exited',
+                };
+                if (sessionName === ORCHESTRATOR_SESSION_NAME) {
+                    terminalGateway.broadcastOrchestratorStatus(statusPayload);
                 }
-            }
-            // Broadcast WebSocket event
-            try {
-                const terminalGateway = getTerminalGateway();
-                if (terminalGateway) {
-                    const statusPayload = {
-                        sessionName,
-                        agentStatus: CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE,
-                        reason: 'runtime_exited',
-                    };
-                    if (sessionName === ORCHESTRATOR_SESSION_NAME) {
-                        terminalGateway.broadcastOrchestratorStatus(statusPayload);
-                    }
-                    else {
-                        terminalGateway.broadcastTeamMemberStatus(statusPayload);
-                    }
+                else {
+                    terminalGateway.broadcastTeamMemberStatus(statusPayload);
                 }
             }
-            catch (error) {
-                this.logger.warn('Failed to broadcast runtime exit event', {
-                    sessionName,
-                    error: error instanceof Error ? error.message : String(error),
-                });
-            }
-            // Cleanup this subscription
-            this.stopMonitoring(sessionName);
         }
-        finally {
-            monitored.confirmationInFlight = false;
+        catch (error) {
+            this.logger.warn('Failed to broadcast runtime exit event', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
         }
     }
     /**
      * Handle a Gemini CLI failure with exponential backoff retry.
      *
      * Waits with increasing delays and checks if the CLI recovers (returns
-     * to a ready prompt). Returns 'recovered' if the CLI is back, or
-     * 'exhausted' if max retries have been reached.
+     * to a ready prompt). Returns 'recovered' if the CLI is back,
+     * 'deferred' if retries remain (skip exit flow, re-check on next
+     * failure detection), or 'exhausted' if max retries have been reached.
      *
      * @param sessionName - PTY session name
      * @param monitored - Monitored session state (retry counter is mutated)
      * @param helper - Session command helper for terminal output capture
-     * @returns 'recovered' if CLI is back, 'exhausted' if retries exhausted
+     * @returns 'recovered' | 'deferred' | 'exhausted'
      */
     async handleGeminiFailureWithRetry(sessionName, monitored, helper) {
         const detectedPattern = GEMINI_FAILURE_PATTERNS.find((p) => p.test(monitored.buffer))?.source;
@@ -496,12 +627,7 @@ export class RuntimeExitMonitorService {
         // Check if the CLI has recovered by looking for ready prompt patterns
         try {
             const output = helper.capturePane(sessionName, RECOVERY_CHECK_LINES);
-            const geminiReadyPatterns = [
-                'Type your message',
-                'gemini>',
-                'context left)',
-            ];
-            const hasRecovered = geminiReadyPatterns.some((pattern) => output.includes(pattern));
+            const hasRecovered = GEMINI_READY_PATTERNS.some((pattern) => output.includes(pattern));
             if (hasRecovered) {
                 this.logger.info('Gemini CLI recovered after transient failure', {
                     sessionName,
@@ -523,9 +649,11 @@ export class RuntimeExitMonitorService {
                 error: error instanceof Error ? error.message : String(error),
             });
         }
-        // Not recovered but retries remain — let the next failure detection handle it
+        // Not recovered but retries remain — defer to the next failure detection cycle.
+        // The buffer was cleared above, so a new failure must appear in terminal
+        // output to re-trigger this method.
         if (monitored.geminiFailureRetries < MAX_RETRIES) {
-            return 'recovered'; // Return 'recovered' to skip exit flow this time
+            return 'deferred';
         }
         return 'exhausted';
     }
@@ -594,7 +722,7 @@ export class RuntimeExitMonitorService {
                 teamId: monitored.teamId,
                 memberId: monitored.memberId,
                 sessionName,
-                agentStatus: 'active',
+                agentStatus: CREWLY_CONSTANTS.AGENT_STATUSES.ACTIVE,
             });
         }
         this.logger.info('Agent restarted after runtime exit with in-progress tasks', {
@@ -728,5 +856,186 @@ export class RuntimeExitMonitorService {
             return false;
         }
     }
+    /**
+     * Check if an agent restart is allowed based on cooldown window.
+     * Prevents infinite restart loops when agents keep crashing (e.g. during
+     * network outages). Uses the same limits as AgentHeartbeatMonitorService.
+     *
+     * @param sessionName - PTY session name
+     * @returns True if restart is allowed, false if cooldown is active
+     */
+    isAgentRestartAllowed(sessionName) {
+        const now = Date.now();
+        const windowStart = now - AGENT_HEARTBEAT_MONITOR_CONSTANTS.COOLDOWN_WINDOW_MS;
+        const timestamps = this.restartHistory.get(sessionName) || [];
+        // Prune old timestamps outside the cooldown window
+        const recent = timestamps.filter(ts => ts > windowStart);
+        this.restartHistory.set(sessionName, recent);
+        return recent.length < AGENT_HEARTBEAT_MONITOR_CONSTANTS.MAX_RESTARTS_PER_WINDOW;
+    }
+    /**
+     * Record a restart attempt for cooldown tracking.
+     *
+     * @param sessionName - PTY session name
+     */
+    recordAgentRestart(sessionName) {
+        const timestamps = this.restartHistory.get(sessionName) || [];
+        timestamps.push(Date.now());
+        this.restartHistory.set(sessionName, timestamps);
+    }
+    /**
+     * Classify an error reason as transient or persistent.
+     *
+     * Transient errors (connectivity, timeout, quota) may resolve on retry.
+     * Persistent errors (auth, config, crash loop) require manual intervention.
+     *
+     * @param reason - The failure reason string
+     * @returns 'TRANSIENT' | 'PERSISTENT' | 'UNKNOWN'
+     */
+    classifyError(reason) {
+        const transientPatterns = [
+            'timeout', 'connectivity', 'connection', 'quota', 'RESOURCE_EXHAUSTED',
+            'UNAVAILABLE', 'DEADLINE_EXCEEDED', 'network', 'INTERNAL',
+        ];
+        const persistentPatterns = [
+            'auth', 'UNAUTHENTICATED', 'PERMISSION_DENIED', 'config',
+            'crash', 'cooldown', 'corruption', 'thinking.*blocks',
+        ];
+        const lowerReason = reason.toLowerCase();
+        if (transientPatterns.some(p => lowerReason.includes(p.toLowerCase()))) {
+            return 'TRANSIENT';
+        }
+        if (persistentPatterns.some(p => new RegExp(p, 'i').test(reason))) {
+            return 'PERSISTENT';
+        }
+        return 'UNKNOWN';
+    }
+    /**
+     * Notify the orchestrator about an agent failure via the chat API (#129).
+     *
+     * Sends a structured failure notification with error classification so the
+     * orchestrator can decide whether to restart the agent, reassign tasks,
+     * or inform the user. This is fire-and-forget — failures are logged but
+     * do not block the exit flow.
+     *
+     * @param sessionName - Failed agent's PTY session name
+     * @param reason - Failure reason (runtime_exited, api_failure, etc.)
+     * @param activeTasks - In-progress tasks the agent had, if any
+     * @param restartSucceeded - Whether automatic restart succeeded
+     */
+    notifyOrchestratorOfFailure(sessionName, reason, activeTasks, restartSucceeded) {
+        const errorClass = this.classifyError(reason);
+        const taskSummary = activeTasks.length > 0
+            ? activeTasks.map(t => `${t.taskName} (${t.status})`).join(', ')
+            : '';
+        const message = [
+            `Agent failure notification: ${sessionName} became inactive.`,
+            `Reason: ${reason}`,
+            `Error classification: ${errorClass}`,
+            activeTasks.length > 0 ? `Active tasks: ${taskSummary}` : 'No active tasks.',
+            restartSucceeded
+                ? 'Automatic restart succeeded — agent is recovering.'
+                : 'Automatic restart FAILED — please restart the agent or reassign its tasks.',
+        ].join('\n');
+        // Send via chat API (fire-and-forget)
+        const backend = getSessionBackendSync();
+        if (!backend)
+            return;
+        const helper = createSessionCommandHelper(backend);
+        const orchestratorSession = helper.getSession(ORCHESTRATOR_SESSION_NAME);
+        if (!orchestratorSession) {
+            this.logger.debug('Orchestrator session not found, skipping failure notification', { sessionName });
+            return;
+        }
+        // Use the deliver endpoint for reliable delivery to the orchestrator
+        const baseUrl = process.env.CREWLY_API_URL || `http://localhost:${WEB_CONSTANTS.PORTS.BACKEND}`;
+        const body = JSON.stringify({
+            message,
+            force: true,
+        });
+        // Fire-and-forget HTTP call
+        try {
+            const url = new URL(`${baseUrl}/api/terminal/${ORCHESTRATOR_SESSION_NAME}/deliver`);
+            const req = http.request(url, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Content-Length': Buffer.byteLength(body).toString(),
+                },
+            });
+            req.on('response', (res) => res.resume()); // Consume response to free socket
+            req.on('error', (err) => {
+                this.logger.debug('Failed to deliver failure notification to orchestrator (non-fatal)', {
+                    sessionName,
+                    error: err.message,
+                });
+            });
+            req.write(body);
+            req.end();
+        }
+        catch {
+            // Non-fatal — notification delivery is best-effort
+        }
+        this.logger.info('Sent agent failure notification to orchestrator (#129)', {
+            sessionName,
+            reason,
+            activeTaskCount: activeTasks.length,
+            restartSucceeded,
+        });
+    }
+    /**
+     * Publish agent:status_changed and agent:inactive events to the EventBus.
+     *
+     * Looks up team/member names from StorageService to build the full
+     * AgentEvent. This mirrors the pattern used in TeamsJsonWatcherService.
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state with teamId/memberId
+     * @param storageService - StorageService instance for team/member lookup
+     */
+    async publishInactiveEvent(sessionName, monitored, storageService) {
+        if (!this.eventBusService) {
+            return;
+        }
+        // Look up team and member names
+        let teamName = '';
+        let memberName = '';
+        try {
+            const result = await storageService.findMemberBySessionName(sessionName);
+            if (result) {
+                teamName = result.team.name;
+                memberName = result.member.name;
+            }
+        }
+        catch {
+            // Best-effort — use empty names if lookup fails
+        }
+        const baseEvent = {
+            id: crypto.randomUUID(),
+            type: 'agent:status_changed',
+            timestamp: new Date().toISOString(),
+            teamId: monitored.teamId || '',
+            teamName,
+            memberId: monitored.memberId || '',
+            memberName,
+            sessionName,
+            previousValue: CREWLY_CONSTANTS.AGENT_STATUSES.ACTIVE,
+            newValue: CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE,
+            changedField: 'agentStatus',
+        };
+        // Publish generic status_changed event
+        this.eventBusService.publish(baseEvent);
+        // Publish specific agent:inactive event
+        this.eventBusService.publish({
+            ...baseEvent,
+            id: crypto.randomUUID(),
+            type: 'agent:inactive',
+        });
+        this.logger.info('Published agent:inactive event to EventBus', {
+            sessionName,
+            teamId: monitored.teamId,
+            memberId: monitored.memberId,
+        });
+    }
 }
 //# sourceMappingURL=runtime-exit-monitor.service.js.map