crewly 1.2.4 → 1.2.6

package/dist/backend/backend/src/services/agent/runtime-exit-monitor.service.js

@@ -185,6 +185,10 @@ export class RuntimeExitMonitorService {
     /**
      * Stop monitoring a PTY session.
      *
+     * Also prunes the restartHistory entry for this session if it has no
+     * recent timestamps within the cooldown window, preventing slow memory
+     * leak for terminated sessions.
+     *
      * @param sessionName - PTY session name
      */
     stopMonitoring(sessionName) {
@@ -200,6 +204,21 @@ export class RuntimeExitMonitorService {
         }
         monitored.unsubscribe();
         this.sessions.delete(sessionName);
+        // Fix 3: Clean up restartHistory for terminated sessions to prevent
+        // unbounded Map growth. Keep the entry only if it has recent timestamps
+        // within the cooldown window (they may still be needed if the session
+        // is restarted soon). Delete the entry entirely otherwise.
+        const timestamps = this.restartHistory.get(sessionName);
+        if (timestamps) {
+            const windowStart = Date.now() - AGENT_HEARTBEAT_MONITOR_CONSTANTS.COOLDOWN_WINDOW_MS;
+            const recent = timestamps.filter(ts => ts > windowStart);
+            if (recent.length === 0) {
+                this.restartHistory.delete(sessionName);
+            }
+            else {
+                this.restartHistory.set(sessionName, recent);
+            }
+        }
         this.logger.debug('Stopped runtime exit monitoring', { sessionName });
     }
     /**
@@ -252,6 +271,10 @@ export class RuntimeExitMonitorService {
     /**
      * Confirm exit by checking for a shell prompt, then react.
      *
+     * Runtime-specific failure checks are gated by runtimeType so that
+     * patterns for one runtime cannot accidentally match another (e.g. a
+     * Codex exit matching Claude or Gemini patterns).
+     *
      * For Gemini CLI failure patterns (API errors, quota exhaustion, etc.),
      * the system retries with exponential backoff before declaring the agent
      * dead. Gemini CLI often recovers automatically from transient API errors
@@ -272,58 +295,11 @@ export class RuntimeExitMonitorService {
             // Verify shell prompt is visible (avoids false positives)
             const shellPromptConfirmed = this.verifyExitWithShellPrompt(sessionName, helper);
             if (!shellPromptConfirmed) {
-                // Codex: "Conversation interrupted" indicates the run was aborted and
-                // often leaves the agent stranded without completion callbacks. Treat it
-                // as an actionable runtime failure even if shell prompt isn't visible.
-                const isCodexInterrupted = monitored.runtimeType === RUNTIME_TYPES.CODEX_CLI
-                    && /Conversation interrupted/i.test(monitored.buffer);
-                if (isCodexInterrupted) {
-                    this.logger.warn('Codex interruption detected, proceeding with recovery flow', {
-                        sessionName,
-                    });
-                    // fall through to exit/restart handling below
-                }
-                // Claude Code: detect fatal API errors that make the CLI permanently
-                // non-functional (e.g. thinking block corruption). These are unrecoverable
-                // — no retry will help, immediate restart is required.
-                const isClaudeFatal = monitored.runtimeType === RUNTIME_TYPES.CLAUDE_CODE
-                    && CLAUDE_FATAL_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
-                if (isClaudeFatal) {
-                    const detectedPattern = CLAUDE_FATAL_PATTERNS.find((p) => p.test(monitored.buffer))?.source;
-                    this.logger.warn('Claude Code fatal error detected, forcing immediate recovery', {
-                        sessionName,
-                        detectedPattern,
-                    });
-                    // fall through to exit/restart handling below (no retry)
-                }
-                // For Gemini CLI: detect either retryable failure patterns or forced
-                // restart/update markers that require immediate recovery handling.
-                const isGeminiFailure = monitored.runtimeType === RUNTIME_TYPES.GEMINI_CLI
-                    && GEMINI_FAILURE_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
-                const isGeminiForceRestart = monitored.runtimeType === RUNTIME_TYPES.GEMINI_CLI
-                    && GEMINI_FORCE_RESTART_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
-                if (!isGeminiFailure && !isGeminiForceRestart && !isCodexInterrupted && !isClaudeFatal) {
+                const runtimeSpecificMatch = await this.checkRuntimeSpecificFailure(sessionName, monitored, helper);
+                if (!runtimeSpecificMatch) {
                     this.logger.debug('Exit pattern matched but shell prompt not confirmed, ignoring', { sessionName });
                     return;
                 }
-                if (isGeminiFailure || isGeminiForceRestart) {
-                    // Some Gemini failures are transient; others (auto-update/restart)
-                    // should trigger immediate recovery and task re-delivery.
-                    if (isGeminiForceRestart) {
-                        this.logger.warn('Gemini auto-update interruption detected, forcing recovery flow', {
-                            sessionName,
-                        });
-                    }
-                    else {
-                        // Retry with exponential backoff before declaring exit.
-                        // Gemini CLI can recover from transient API errors automatically.
-                        const retryResult = await this.handleGeminiFailureWithRetry(sessionName, monitored, helper);
-                        if (retryResult === 'recovered' || retryResult === 'deferred') {
-                            return;
-                        }
-                        // retryResult === 'exhausted' — fall through to exit flow
-                    }
-                }
             }
             // Mark as detected to prevent double-processing
             monitored.exitDetected = true;
@@ -333,199 +309,281 @@ export class RuntimeExitMonitorService {
                 role: monitored.role,
             });
             // Fire the exit-detected callback (used to cancel pending registrations)
-            if (this.onExitDetectedCallback) {
-                try {
-                    this.onExitDetectedCallback(sessionName);
-                }
-                catch (error) {
-                    this.logger.warn('onExitDetected callback error', {
-                        sessionName,
-                        error: error instanceof Error ? error.message : String(error),
-                    });
-                }
-            }
-            // Check for in-progress tasks — restart agent if any exist (non-orchestrator only)
+            this.fireExitDetectedCallback(sessionName);
+            // Try agent restart if it has in-progress tasks (non-orchestrator only)
             if (monitored.role !== ORCHESTRATOR_ROLE && this.agentRegistrationService && this.taskTrackingService) {
-                try {
-                    const tasks = await this.taskTrackingService.getTasksForTeamMember(monitored.memberId || '');
-                    const activeTasks = tasks.filter(t => t.status === 'assigned' || t.status === 'active' || t.status === 'pending_assignment');
-                    if (activeTasks.length > 0) {
-                        // Check cooldown before attempting restart to prevent infinite
-                        // restart loops (e.g. during network outages where agents keep crashing)
-                        if (!this.isAgentRestartAllowed(sessionName)) {
-                            this.logger.warn('Agent restart cooldown active, skipping restart', {
-                                sessionName,
-                                activeTaskCount: activeTasks.length,
-                                maxRestartsPerWindow: AGENT_HEARTBEAT_MONITOR_CONSTANTS.MAX_RESTARTS_PER_WINDOW,
-                                cooldownWindowMs: AGENT_HEARTBEAT_MONITOR_CONSTANTS.COOLDOWN_WINDOW_MS,
-                            });
-                            this.notifyOrchestratorOfFailure(sessionName, 'runtime_exited_cooldown', activeTasks, false);
-                            // Fall through to normal inactive flow
-                        }
-                        else {
-                            this.logger.info('Runtime exit with in-progress tasks, attempting restart', {
-                                sessionName,
-                                activeTaskCount: activeTasks.length,
-                            });
-                            try {
-                                await this.restartAgentWithTasks(sessionName, monitored, activeTasks);
-                                this.recordAgentRestart(sessionName);
-                                // Notify orchestrator that agent was restarted (#129)
-                                this.notifyOrchestratorOfFailure(sessionName, 'runtime_exited', activeTasks, true);
-                                // Cleanup this subscription (restart succeeded, skip inactive flow)
-                                this.stopMonitoring(sessionName);
-                                return;
-                            }
-                            catch (restartError) {
-                                this.logger.warn('Agent restart after runtime exit failed, falling back to inactive', {
-                                    sessionName,
-                                    error: restartError instanceof Error ? restartError.message : String(restartError),
-                                });
-                                // Record the attempt even on failure to prevent rapid retry loops
-                                this.recordAgentRestart(sessionName);
-                                // Notify orchestrator that restart failed — needs manual intervention (#129)
-                                this.notifyOrchestratorOfFailure(sessionName, 'runtime_exited', activeTasks, false);
-                                // Fall through to normal inactive flow
-                            }
-                        }
-                    }
-                }
-                catch (error) {
-                    this.logger.warn('Failed to check in-progress tasks after runtime exit', {
-                        sessionName,
-                        error: error instanceof Error ? error.message : String(error),
-                    });
-                    // Fall through to normal inactive flow
-                }
+                const restarted = await this.tryAgentRestartWithTasks(sessionName, monitored);
+                if (restarted)
+                    return;
             }
-            // Orchestrator-specific restart: attempt auto-restart when runtime exits
+            // Orchestrator-specific restart
             if (sessionName === ORCHESTRATOR_SESSION_NAME) {
-                // CRITICAL: Set status to inactive BEFORE restart so the queue
-                // processor doesn't deliver messages during postInitialize.
-                // Without this, the old 'active' status persists in storage and
-                // the queue processor sends chat messages that get concatenated
-                // into /directory add commands in the new Gemini CLI session.
-                try {
-                    const storageService = StorageService.getInstance();
-                    await storageService.updateAgentStatus(sessionName, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE);
-                }
-                catch {
-                    // Best-effort — continue with restart
-                }
-                // Capture session memory before restart
-                try {
-                    const sessionMemoryService = SessionMemoryService.getInstance();
-                    await sessionMemoryService.onSessionEnd(sessionName, monitored.role, process.cwd());
-                }
-                catch (error) {
-                    this.logger.warn('Failed to capture session memory before orchestrator restart', {
-                        sessionName,
-                        error: error instanceof Error ? error.message : String(error),
-                    });
+                const restarted = await this.tryOrchestratorRestart(sessionName, monitored);
+                if (restarted)
+                    return;
+            }
+            // Normal inactive flow: update status, capture memory, broadcast
+            await this.transitionToInactive(sessionName, monitored);
+            // Cleanup this subscription
+            this.stopMonitoring(sessionName);
+        }
+        finally {
+            monitored.confirmationInFlight = false;
+        }
+    }
+    /**
+     * Check for runtime-specific failure patterns when shell prompt is not confirmed.
+     *
+     * Gated by runtimeType so patterns for one runtime cannot accidentally
+     * match another. Returns true if a runtime-specific failure was detected,
+     * or 'deferred' handling occurred (Gemini retry).
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state
+     * @param helper - Session command helper
+     * @returns True if a runtime-specific failure was confirmed
+     */
+    async checkRuntimeSpecificFailure(sessionName, monitored, helper) {
+        if (monitored.runtimeType === RUNTIME_TYPES.CODEX_CLI) {
+            const isCodexInterrupted = /Conversation interrupted/i.test(monitored.buffer);
+            if (isCodexInterrupted) {
+                this.logger.warn('Codex interruption detected, proceeding with recovery flow', { sessionName });
+                return true;
+            }
+        }
+        else if (monitored.runtimeType === RUNTIME_TYPES.CLAUDE_CODE) {
+            const isClaudeFatal = CLAUDE_FATAL_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
+            if (isClaudeFatal) {
+                const detectedPattern = CLAUDE_FATAL_PATTERNS.find((p) => p.test(monitored.buffer))?.source;
+                this.logger.warn('Claude Code fatal error detected, forcing immediate recovery', {
+                    sessionName,
+                    detectedPattern,
+                });
+                return true;
+            }
+        }
+        else if (monitored.runtimeType === RUNTIME_TYPES.GEMINI_CLI) {
+            const isGeminiFailure = GEMINI_FAILURE_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
+            const isGeminiForceRestart = GEMINI_FORCE_RESTART_PATTERNS.some((pattern) => pattern.test(monitored.buffer));
+            if (isGeminiFailure || isGeminiForceRestart) {
+                if (isGeminiForceRestart) {
+                    this.logger.warn('Gemini auto-update interruption detected, forcing recovery flow', { sessionName });
                 }
-                try {
-                    const restartService = OrchestratorRestartService.getInstance();
-                    const success = await restartService.attemptRestart();
-                    if (success) {
-                        this.logger.info('Orchestrator restarted after runtime exit', { sessionName });
-                        // Broadcast restarted status as STARTED (not ACTIVE).
-                        // The agent becomes ACTIVE only after the registration prompt
-                        // is fully processed and the MCP registration call completes.
-                        // Broadcasting ACTIVE prematurely causes the queue processor
-                        // to deliver messages during postInitialize.
-                        try {
-                            const terminalGateway = getTerminalGateway();
-                            if (terminalGateway) {
-                                terminalGateway.broadcastOrchestratorStatus({
-                                    sessionName,
-                                    agentStatus: CREWLY_CONSTANTS.AGENT_STATUSES.STARTED,
-                                    reason: 'runtime_exit_restart',
-                                });
-                            }
-                        }
-                        catch {
-                            // Best-effort broadcast
-                        }
-                        this.stopMonitoring(sessionName);
-                        return;
+                else {
+                    const retryResult = await this.handleGeminiFailureWithRetry(sessionName, monitored, helper);
+                    if (retryResult === 'recovered' || retryResult === 'deferred') {
+                        return false; // Not a confirmed exit — caller should return early
                     }
-                    this.logger.warn('Orchestrator restart after runtime exit failed or not allowed, falling back to inactive', {
-                        sessionName,
-                    });
-                }
-                catch (error) {
-                    this.logger.warn('Orchestrator restart attempt threw error, falling back to inactive', {
-                        sessionName,
-                        error: error instanceof Error ? error.message : String(error),
-                    });
                 }
-                // Fall through to normal inactive flow
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Fire the onExitDetected callback safely.
+     *
+     * @param sessionName - PTY session name
+     */
+    fireExitDetectedCallback(sessionName) {
+        if (!this.onExitDetectedCallback)
+            return;
+        try {
+            this.onExitDetectedCallback(sessionName);
+        }
+        catch (error) {
+            this.logger.warn('onExitDetected callback error', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    /**
+     * Try to restart an agent that has in-progress tasks.
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state
+     * @returns True if the agent was successfully restarted (caller should return)
+     */
+    async tryAgentRestartWithTasks(sessionName, monitored) {
+        try {
+            const tasks = await this.taskTrackingService.getTasksForTeamMember(monitored.memberId || '');
+            const activeTasks = tasks.filter(t => t.status === 'assigned' || t.status === 'active' || t.status === 'pending_assignment');
+            if (activeTasks.length === 0)
+                return false;
+            if (!this.isAgentRestartAllowed(sessionName)) {
+                this.logger.warn('Agent restart cooldown active, skipping restart', {
+                    sessionName,
+                    activeTaskCount: activeTasks.length,
+                    maxRestartsPerWindow: AGENT_HEARTBEAT_MONITOR_CONSTANTS.MAX_RESTARTS_PER_WINDOW,
+                    cooldownWindowMs: AGENT_HEARTBEAT_MONITOR_CONSTANTS.COOLDOWN_WINDOW_MS,
+                });
+                this.notifyOrchestratorOfFailure(sessionName, 'runtime_exited_cooldown', activeTasks, false);
+                return false;
             }
-            // Update agent status to inactive
+            this.logger.info('Runtime exit with in-progress tasks, attempting restart', {
+                sessionName,
+                activeTaskCount: activeTasks.length,
+            });
             try {
-                const storageService = StorageService.getInstance();
-                await storageService.updateAgentStatus(sessionName, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE);
-                this.logger.info('Agent status updated to inactive after runtime exit', { sessionName });
-                // Publish agent:inactive event to EventBus so orchestrator is notified
-                if (this.eventBusService) {
-                    try {
-                        await this.publishInactiveEvent(sessionName, monitored, storageService);
-                    }
-                    catch (eventError) {
-                        this.logger.warn('Failed to publish agent:inactive event to EventBus', {
+                await this.restartAgentWithTasks(sessionName, monitored, activeTasks);
+                this.recordAgentRestart(sessionName);
+                this.notifyOrchestratorOfFailure(sessionName, 'runtime_exited', activeTasks, true);
+                this.stopMonitoring(sessionName);
+                return true;
+            }
+            catch (restartError) {
+                this.logger.warn('Agent restart after runtime exit failed, falling back to inactive', {
+                    sessionName,
+                    error: restartError instanceof Error ? restartError.message : String(restartError),
+                });
+                this.recordAgentRestart(sessionName);
+                this.notifyOrchestratorOfFailure(sessionName, 'runtime_exited', activeTasks, false);
+                return false;
+            }
+        }
+        catch (error) {
+            this.logger.warn('Failed to check in-progress tasks after runtime exit', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return false;
+        }
+    }
+    /**
+     * Try to restart the orchestrator after runtime exit.
+     *
+     * Sets status to inactive before restart to prevent queue processor
+     * from delivering messages during postInitialize, captures session
+     * memory, and broadcasts restart status.
+     *
+     * @param sessionName - PTY session name (should be ORCHESTRATOR_SESSION_NAME)
+     * @param monitored - Monitored session state
+     * @returns True if orchestrator was successfully restarted (caller should return)
+     */
+    async tryOrchestratorRestart(sessionName, monitored) {
+        // Set status to inactive BEFORE restart so the queue processor
+        // doesn't deliver messages during postInitialize.
+        try {
+            const storageService = StorageService.getInstance();
+            await storageService.updateAgentStatus(sessionName, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE);
+        }
+        catch {
+            // Best-effort — continue with restart
+        }
+        // Capture session memory before restart
+        try {
+            const sessionMemoryService = SessionMemoryService.getInstance();
+            await sessionMemoryService.onSessionEnd(sessionName, monitored.role, process.cwd());
+        }
+        catch (error) {
+            this.logger.warn('Failed to capture session memory before orchestrator restart', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+        try {
+            const restartService = OrchestratorRestartService.getInstance();
+            const success = await restartService.attemptRestart();
+            if (success) {
+                this.logger.info('Orchestrator restarted after runtime exit', { sessionName });
+                // Broadcast STARTED (not ACTIVE) — agent becomes ACTIVE only
+                // after registration prompt is processed and MCP call completes.
+                try {
+                    const terminalGateway = getTerminalGateway();
+                    if (terminalGateway) {
+                        terminalGateway.broadcastOrchestratorStatus({
                             sessionName,
-                            error: eventError instanceof Error ? eventError.message : String(eventError),
+                            agentStatus: CREWLY_CONSTANTS.AGENT_STATUSES.STARTED,
+                            reason: 'runtime_exit_restart',
                         });
                     }
                 }
+                catch {
+                    // Best-effort broadcast
+                }
+                this.stopMonitoring(sessionName);
+                return true;
             }
-            catch (error) {
-                this.logger.warn('Failed to update agent status after runtime exit', {
-                    sessionName,
-                    error: error instanceof Error ? error.message : String(error),
-                });
-            }
-            // Capture session memory (skip for orchestrator — already captured above)
-            if (sessionName !== ORCHESTRATOR_SESSION_NAME) {
+            this.logger.warn('Orchestrator restart after runtime exit failed or not allowed, falling back to inactive', {
+                sessionName,
+            });
+        }
+        catch (error) {
+            this.logger.warn('Orchestrator restart attempt threw error, falling back to inactive', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+        return false;
+    }
+    /**
+     * Transition agent to inactive state: update storage, capture memory,
+     * publish events, and broadcast WebSocket status.
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state
+     */
+    async transitionToInactive(sessionName, monitored) {
+        // Update agent status to inactive
+        try {
+            const storageService = StorageService.getInstance();
+            await storageService.updateAgentStatus(sessionName, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE);
+            this.logger.info('Agent status updated to inactive after runtime exit', { sessionName });
+            // Publish agent:inactive event to EventBus so orchestrator is notified
+            if (this.eventBusService) {
                 try {
-                    const sessionMemoryService = SessionMemoryService.getInstance();
-                    await sessionMemoryService.onSessionEnd(sessionName, monitored.role, process.cwd());
+                    await this.publishInactiveEvent(sessionName, monitored, storageService);
                 }
-                catch (error) {
-                    this.logger.warn('Failed to capture session memory after runtime exit', {
+                catch (eventError) {
+                    this.logger.warn('Failed to publish agent:inactive event to EventBus', {
                         sessionName,
-                        error: error instanceof Error ? error.message : String(error),
+                        error: eventError instanceof Error ? eventError.message : String(eventError),
                     });
                 }
             }
-            // Broadcast WebSocket event
+        }
+        catch (error) {
+            this.logger.warn('Failed to update agent status after runtime exit', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+        // Capture session memory (skip for orchestrator — already captured above)
+        if (sessionName !== ORCHESTRATOR_SESSION_NAME) {
             try {
-                const terminalGateway = getTerminalGateway();
-                if (terminalGateway) {
-                    const statusPayload = {
-                        sessionName,
-                        agentStatus: CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE,
-                        reason: 'runtime_exited',
-                    };
-                    if (sessionName === ORCHESTRATOR_SESSION_NAME) {
-                        terminalGateway.broadcastOrchestratorStatus(statusPayload);
-                    }
-                    else {
-                        terminalGateway.broadcastTeamMemberStatus(statusPayload);
-                    }
-                }
+                const sessionMemoryService = SessionMemoryService.getInstance();
+                await sessionMemoryService.onSessionEnd(sessionName, monitored.role, process.cwd());
             }
             catch (error) {
-                this.logger.warn('Failed to broadcast runtime exit event', {
+                this.logger.warn('Failed to capture session memory after runtime exit', {
                     sessionName,
                     error: error instanceof Error ? error.message : String(error),
                 });
             }
-            // Cleanup this subscription
-            this.stopMonitoring(sessionName);
         }
-        finally {
-            monitored.confirmationInFlight = false;
+        // Broadcast WebSocket event
+        try {
+            const terminalGateway = getTerminalGateway();
+            if (terminalGateway) {
+                const statusPayload = {
+                    sessionName,
+                    agentStatus: CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE,
+                    reason: 'runtime_exited',
+                };
+                if (sessionName === ORCHESTRATOR_SESSION_NAME) {
+                    terminalGateway.broadcastOrchestratorStatus(statusPayload);
+                }
+                else {
+                    terminalGateway.broadcastTeamMemberStatus(statusPayload);
+                }
+            }
+        }
+        catch (error) {
+            this.logger.warn('Failed to broadcast runtime exit event', {
+                sessionName,
+                error: error instanceof Error ? error.message : String(error),
+            });
         }
     }
     /**