crewly 1.13.0 → 1.14.0

package/dist/cli/backend/src/services/agent/runtime-exit-monitor.service.d.ts

@@ -0,0 +1,295 @@
+import type { AgentRegistrationService } from './agent-registration.service.js';
+import type { EventBusService } from '../event-bus/event-bus.service.js';
+import { type RuntimeType } from '../../constants.js';
+/**
+ * Service that monitors PTY sessions for runtime exit patterns.
+ *
+ * When an agent CLI (Claude Code, Gemini CLI, Codex CLI) exits inside a PTY
+ * session, the PTY shell itself stays alive. This service watches terminal
+ * output for runtime-specific exit patterns and reacts by:
+ *
+ * 1. Updating the agent status to `inactive`
+ * 2. Capturing session memory
+ * 3. Broadcasting WebSocket events to the frontend
+ * 4. Cancelling pending registration operations
+ *
+ * For Gemini CLI, the service also detects failure patterns (API errors,
+ * quota exhaustion, network issues) that indicate the CLI is stuck and
+ * needs recovery. These failure patterns bypass the shell prompt verification
+ * since the CLI may still be running but non-functional.
+ *
+ * For Claude Code, the service detects fatal API errors (e.g. thinking
+ * block corruption) that make the CLI permanently non-functional within
+ * the current session. These trigger immediate recovery without retry.
+ *
+ * @example
+ * ```typescript
+ * const monitor = RuntimeExitMonitorService.getInstance();
+ * monitor.startMonitoring('agent-dev-001', 'claude-code', 'developer');
+ * // ... later ...
+ * monitor.stopMonitoring('agent-dev-001');
+ * ```
+ */
+export declare class RuntimeExitMonitorService {
+    private static instance;
+    private logger;
+    private sessions;
+    private onExitDetectedCallback?;
+    private agentRegistrationService;
+    private eventBusService;
+    /**
+     * Restart timestamps per session, persisted across monitoring cycles.
+     * Unlike MonitoredSession (which is recreated on each startMonitoring call),
+     * this map survives agent restarts to prevent infinite restart loops
+     * when agents keep crashing (e.g. during network outages).
+     */
+    private restartHistory;
+    private constructor();
+    /**
+     * Get the singleton instance.
+     */
+    static getInstance(): RuntimeExitMonitorService;
+    /**
+     * Reset the singleton (for testing).
+     */
+    static resetInstance(): void;
+    /**
+     * Register a callback invoked when a runtime exit is detected.
+     * Used by AgentRegistrationService to cancel pending registrations.
+     *
+     * @param callback - Called with the sessionName on exit detection
+     */
+    setOnExitDetectedCallback(callback: (sessionName: string) => void): void;
+    /**
+     * Set the AgentRegistrationService dependency for agent restart.
+     *
+     * @param service - The AgentRegistrationService instance
+     */
+    setAgentRegistrationService(service: AgentRegistrationService): void;
+    /**
+     * Backwards-compatible no-op. Kept so `index.ts` compiles during the
+     * TaskTrackingService deletion window. In-progress task queries now
+     * read directly from TaskPoolService.
+     *
+     * @deprecated Will be removed in a follow-up. Do not call from new code.
+     */
+    setTaskTrackingService(_service: unknown): void;
+    /**
+     * Set the EventBusService dependency for publishing agent lifecycle events.
+     *
+     * @param service - The EventBusService instance
+     */
+    setEventBusService(service: EventBusService): void;
+    /**
+     * Start monitoring a PTY session for runtime exit patterns.
+     *
+     * @param sessionName - PTY session name
+     * @param runtimeType - Agent runtime type (claude-code, gemini-cli, codex-cli)
+     * @param role - Agent role name
+     * @param teamId - Optional team identifier
+     * @param memberId - Optional member identifier
+     */
+    startMonitoring(sessionName: string, runtimeType: RuntimeType, role: string, teamId?: string, memberId?: string): void;
+    /**
+     * Stop monitoring a PTY session.
+     *
+     * Also prunes the restartHistory entry for this session if it has no
+     * recent timestamps within the cooldown window, preventing slow memory
+     * leak for terminated sessions.
+     *
+     * @param sessionName - PTY session name
+     */
+    stopMonitoring(sessionName: string): void;
+    /**
+     * Check if a session is being monitored.
+     */
+    isMonitoring(sessionName: string): boolean;
+    /**
+     * Destroy all monitoring subscriptions.
+     */
+    destroy(): void;
+    /**
+     * Handle incoming PTY data for a monitored session.
+     */
+    private handleData;
+    /**
+     * Confirm exit by checking for a shell prompt, then react.
+     *
+     * Runtime-specific failure checks are gated by runtimeType so that
+     * patterns for one runtime cannot accidentally match another (e.g. a
+     * Codex exit matching Claude or Gemini patterns).
+     *
+     * For Gemini CLI failure patterns (API errors, quota exhaustion, etc.),
+     * the system retries with exponential backoff before declaring the agent
+     * dead. Gemini CLI often recovers automatically from transient API errors
+     * (RESOURCE_EXHAUSTED, UNAVAILABLE, Connection error). Only after
+     * MAX_RETRIES does the system proceed with the exit/restart flow.
+     *
+     * For Claude Code fatal patterns (thinking block corruption, etc.),
+     * the system skips retry and forces immediate recovery since these
+     * errors are permanently unrecoverable within the same session.
+     */
+    private confirmAndReact;
+    /**
+     * Check for runtime-specific failure patterns when shell prompt is not confirmed.
+     *
+     * Gated by runtimeType so patterns for one runtime cannot accidentally
+     * match another. Returns true if a runtime-specific failure was detected,
+     * or 'deferred' handling occurred (Gemini retry).
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state
+     * @param helper - Session command helper
+     * @returns True if a runtime-specific failure was confirmed
+     */
+    private checkRuntimeSpecificFailure;
+    /**
+     * Fire the onExitDetected callback safely.
+     *
+     * @param sessionName - PTY session name
+     */
+    private fireExitDetectedCallback;
+    /**
+     * Try to restart an agent that has in-progress tasks.
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state
+     * @returns True if the agent was successfully restarted (caller should return)
+     */
+    private tryAgentRestartWithTasks;
+    /**
+     * Try to restart the orchestrator after runtime exit.
+     *
+     * Sets status to inactive before restart to prevent queue processor
+     * from delivering messages during postInitialize, captures session
+     * memory, and broadcasts restart status.
+     *
+     * @param sessionName - PTY session name (should be ORCHESTRATOR_SESSION_NAME)
+     * @param monitored - Monitored session state
+     * @returns True if orchestrator was successfully restarted (caller should return)
+     */
+    private tryOrchestratorRestart;
+    /**
+     * Transition agent to inactive state: update storage, capture memory,
+     * publish events, and broadcast WebSocket status.
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state
+     */
+    private transitionToInactive;
+    /**
+     * Handle a Gemini CLI failure with exponential backoff retry.
+     *
+     * Waits with increasing delays and checks if the CLI recovers (returns
+     * to a ready prompt). Returns 'recovered' if the CLI is back,
+     * 'deferred' if retries remain (skip exit flow, re-check on next
+     * failure detection), or 'exhausted' if max retries have been reached.
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state (retry counter is mutated)
+     * @param helper - Session command helper for terminal output capture
+     * @returns 'recovered' | 'deferred' | 'exhausted'
+     */
+    private handleGeminiFailureWithRetry;
+    /**
+     * Restart an agent that exited while it had in-progress tasks.
+     *
+     * Mirrors the restart pattern from AgentHeartbeatMonitorService:
+     * 1. Save Claude session ID for resume
+     * 2. Kill old PTY session + clear PTY activity tracker
+     * 3. Pre-set session ID for resume
+     * 4. Recreate agent session via AgentRegistrationService
+     * 5. Broadcast restarted status via WebSocket
+     * 6. Re-deliver tasks asynchronously
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state
+     * @param activeTasks - In-progress tasks to re-deliver
+     */
+    private restartAgentWithTasks;
+    /**
+     * Re-deliver in-progress tasks to a restarted agent.
+     *
+     * Waits for the agent to initialize, reads task files, and writes
+     * summaries into the agent's PTY session.
+     *
+     * @param sessionName - PTY session name
+     * @param activeTasks - Tasks to re-deliver
+     */
+    private redeliverTasks;
+    /**
+     * Periodic check: is the runtime child process still alive?
+     *
+     * Uses the session backend's isChildProcessAlive() (pgrep -P <pid>)
+     * to determine if the CLI process has exited. This is a version-agnostic
+     * fallback that doesn't depend on any text the CLI prints.
+     *
+     * @param sessionName - PTY session name
+     * @param backend - Session backend for process checks
+     * @param helper - Session command helper for shell prompt verification
+     */
+    private checkProcessAlive;
+    /**
+     * Verify that a shell prompt is visible in the terminal, confirming
+     * that the agent CLI has actually exited and returned to the shell.
+     */
+    private verifyExitWithShellPrompt;
+    /**
+     * Check if an agent restart is allowed based on cooldown window.
+     * Prevents infinite restart loops when agents keep crashing (e.g. during
+     * network outages). Uses the same limits as AgentHeartbeatMonitorService.
+     *
+     * @param sessionName - PTY session name
+     * @returns True if restart is allowed, false if cooldown is active
+     */
+    private isAgentRestartAllowed;
+    /**
+     * Record a restart attempt for cooldown tracking.
+     *
+     * @param sessionName - PTY session name
+     */
+    private recordAgentRestart;
+    /**
+     * Classify an error reason as transient or persistent.
+     *
+     * Transient errors (connectivity, timeout, quota) may resolve on retry.
+     * Persistent errors (auth, config, crash loop) require manual intervention.
+     *
+     * @param reason - The failure reason string
+     * @returns 'TRANSIENT' | 'PERSISTENT' | 'UNKNOWN'
+     */
+    private classifyError;
+    /**
+     * Notify the orchestrator about an agent failure via the chat API (#129).
+     *
+     * Sends a structured failure notification with error classification so the
+     * orchestrator can decide whether to restart the agent, reassign tasks,
+     * or inform the user. This is fire-and-forget — failures are logged but
+     * do not block the exit flow.
+     *
+     * @param sessionName - Failed agent's PTY session name
+     * @param reason - Failure reason (runtime_exited, api_failure, etc.)
+     * @param activeTasks - In-progress tasks the agent had, if any
+     * @param restartSucceeded - Whether automatic restart succeeded
+     */
+    private notifyOrchestratorOfFailure;
+    /**
+     * #235: Infer dropout reason from the monitored session's buffer.
+     *
+     * @param monitored - Monitored session state
+     * @returns Inferred dropout reason
+     */
+    private inferDropoutReason;
+    /**
+     * Publish agent:status_changed and agent:inactive events to the EventBus.
+     *
+     * Looks up team/member names from StorageService to build the full
+     * AgentEvent. This mirrors the pattern used in TeamsJsonWatcherService.
+     *
+     * @param sessionName - PTY session name
+     * @param monitored - Monitored session state with teamId/memberId
+     * @param storageService - StorageService instance for team/member lookup
+     */
+    private publishInactiveEvent;
+}
+//# sourceMappingURL=runtime-exit-monitor.service.d.ts.map

package/dist/cli/backend/src/services/agent/runtime-exit-monitor.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime-exit-monitor.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/agent/runtime-exit-monitor.service.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAC;AAEhF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AAGzE,OAAO,EAoBN,KAAK,WAAW,EAChB,MAAM,oBAAoB,CAAC;AA2B5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,yBAAyB;IACrC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAA0C;IACjE,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,QAAQ,CAAuC;IACvD,OAAO,CAAC,sBAAsB,CAAC,CAAgC;IAC/D,OAAO,CAAC,wBAAwB,CAAyC;IACzE,OAAO,CAAC,eAAe,CAAgC;IAEvD;;;;;OAKG;IACH,OAAO,CAAC,cAAc,CAA+B;IAErD,OAAO;IAIP;;OAEG;IACH,MAAM,CAAC,WAAW,IAAI,yBAAyB;IAO/C;;OAEG;IACH,MAAM,CAAC,aAAa,IAAI,IAAI;IAO5B;;;;;OAKG;IACH,yBAAyB,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;IAIxE;;;;OAIG;IACH,2BAA2B,CAAC,OAAO,EAAE,wBAAwB,GAAG,IAAI;IAIpE;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI;IAI/C;;;;OAIG;IACH,kBAAkB,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAIlD;;;;;;;;OAQG;IACH,eAAe,CACd,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GACf,IAAI;IAyEP;;;;;;;;OAQG;IACH,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAkCzC;;OAEG;IACH,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IAI1C;;OAEG;IACH,OAAO,IAAI,IAAI;IASf;;OAEG;IACH,OAAO,CAAC,UAAU;IAkFlB;;;;;;;;;;;;;;;;OAgBG;YACW,eAAe;IA2F7B;;;;;;;;;;;OAWG;YACW,2BAA2B;IAsDzC;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAYhC;;;;;;OAMG;YACW,wBAAwB;IA+DtC;;;;;;;;;;OAUG;YACW,sBAAsB;IA+DpC;;;;;;OAMG;YACW,oBAAoB;IAyElC;;;;;;;;;;;;OAYG;YACW,4BAA4B;IAkF1C;;;;;;;;;;;;;;OAcG;YACW,qBAAqB;IA8EnC;;;;;;;;OAQG;YACW,cAAc;IAkE5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,iBAAiB;IA6CzB;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IASjC;;;;;;;OAOG;IACH,OAAO,CAAC,qBAAqB;IAY7B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAM1B;;;;;;;;OAQG;IACH,OAAO,CAAC,aAAa;IAoBrB;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,2BAA2B;IAsEnC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IA0B1B;;;;;;;;;OASG;YACW,oBAAoB;CAoDlC"}