instar 0.7.52 → 0.7.53

package/dist/monitoring/SessionWatchdog.js

@@ -0,0 +1,326 @@
+/**
+ * SessionWatchdog — Auto-remediation for stuck Claude sessions (Instar port).
+ *
+ * Detects when a Claude session has a long-running bash command and escalates
+ * from gentle (Ctrl+C) to forceful (SIGKILL + session kill). Adapted from
+ * Dawn Server's SessionWatchdog for Instar's self-contained architecture.
+ *
+ * Escalation pipeline:
+ *   Level 0: Monitoring (default)
+ *   Level 1: Ctrl+C via tmux send-keys
+ *   Level 2: SIGTERM the stuck child PID
+ *   Level 3: SIGKILL the stuck child PID
+ *   Level 4: Kill tmux session
+ */
+import { execSync } from 'node:child_process';
+import { EventEmitter } from 'node:events';
+export var EscalationLevel;
+(function (EscalationLevel) {
+    EscalationLevel[EscalationLevel["Monitoring"] = 0] = "Monitoring";
+    EscalationLevel[EscalationLevel["CtrlC"] = 1] = "CtrlC";
+    EscalationLevel[EscalationLevel["SigTerm"] = 2] = "SigTerm";
+    EscalationLevel[EscalationLevel["SigKill"] = 3] = "SigKill";
+    EscalationLevel[EscalationLevel["KillSession"] = 4] = "KillSession";
+})(EscalationLevel || (EscalationLevel = {}));
+// Processes that are long-running by design
+const EXCLUDED_PATTERNS = [
+    'playwright-mcp', 'playwright-persistent', '@playwright/mcp',
+    'chrome-native-host', 'claude-in-chrome-mcp', 'payments-mcp',
+    'mcp-remote', '/mcp/', '.mcp/', 'caffeinate', 'exa-mcp-server',
+];
+const EXCLUDED_PREFIXES = [
+    '/bin/zsh -c -l source',
+    '/bin/bash -c -l source',
+];
+// Escalation delays (ms to wait before advancing to next level)
+const ESCALATION_DELAYS = {
+    [EscalationLevel.Monitoring]: 0,
+    [EscalationLevel.CtrlC]: 0,
+    [EscalationLevel.SigTerm]: 15_000,
+    [EscalationLevel.SigKill]: 10_000,
+    [EscalationLevel.KillSession]: 5_000,
+};
+const DEFAULT_STUCK_THRESHOLD_MS = 180_000; // 3 minutes
+const DEFAULT_POLL_INTERVAL_MS = 30_000;
+const MAX_RETRIES = 2;
+export class SessionWatchdog extends EventEmitter {
+    config;
+    sessionManager;
+    state;
+    interval = null;
+    escalationState = new Map();
+    interventionHistory = [];
+    enabled = true;
+    running = false;
+    stuckThresholdMs;
+    pollIntervalMs;
+    constructor(config, sessionManager, state) {
+        super();
+        this.config = config;
+        this.sessionManager = sessionManager;
+        this.state = state;
+        const wdConfig = config.monitoring.watchdog;
+        this.stuckThresholdMs = (wdConfig?.stuckCommandSec ?? 180) * 1000;
+        this.pollIntervalMs = wdConfig?.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+    }
+    start() {
+        if (this.interval)
+            return;
+        console.log(`[Watchdog] Starting (poll: ${this.pollIntervalMs / 1000}s, threshold: ${this.stuckThresholdMs / 1000}s)`);
+        this.interval = setInterval(() => this.poll(), this.pollIntervalMs);
+        setTimeout(() => this.poll(), 5000);
+    }
+    stop() {
+        if (this.interval) {
+            clearInterval(this.interval);
+            this.interval = null;
+        }
+    }
+    setEnabled(enabled) {
+        this.enabled = enabled;
+        if (!enabled) {
+            this.escalationState.clear();
+        }
+    }
+    isEnabled() {
+        return this.enabled;
+    }
+    isManaging(sessionName) {
+        const s = this.escalationState.get(sessionName);
+        return s !== undefined && s.level > EscalationLevel.Monitoring;
+    }
+    getStatus() {
+        const runningSessions = this.sessionManager.listRunningSessions();
+        const sessions = runningSessions.map(s => ({
+            name: s.tmuxSession,
+            escalation: this.escalationState.get(s.tmuxSession) ?? null,
+        }));
+        return {
+            enabled: this.enabled,
+            sessions,
+            interventionHistory: this.interventionHistory.slice(-20),
+        };
+    }
+    // --- Core polling ---
+    async poll() {
+        if (!this.enabled || this.running)
+            return;
+        this.running = true;
+        try {
+            const sessions = this.sessionManager.listRunningSessions();
+            for (const session of sessions) {
+                try {
+                    this.checkSession(session.tmuxSession);
+                }
+                catch (err) {
+                    console.error(`[Watchdog] Error checking "${session.tmuxSession}":`, err);
+                }
+            }
+        }
+        finally {
+            this.running = false;
+        }
+    }
+    checkSession(tmuxSession) {
+        const existing = this.escalationState.get(tmuxSession);
+        if (existing && existing.level > EscalationLevel.Monitoring) {
+            this.handleEscalation(tmuxSession, existing);
+            return;
+        }
+        // Find Claude PID in the tmux session
+        const claudePid = this.getClaudePid(tmuxSession);
+        if (!claudePid)
+            return;
+        const children = this.getChildProcesses(claudePid);
+        const stuckChild = children.find(c => !this.isExcluded(c.command) && c.elapsedMs > this.stuckThresholdMs);
+        if (stuckChild) {
+            const state = {
+                level: EscalationLevel.CtrlC,
+                levelEnteredAt: Date.now(),
+                stuckChildPid: stuckChild.pid,
+                stuckCommand: stuckChild.command,
+                retryCount: existing?.retryCount ?? 0,
+            };
+            this.escalationState.set(tmuxSession, state);
+            console.log(`[Watchdog] "${tmuxSession}": stuck command (${Math.round(stuckChild.elapsedMs / 1000)}s): ` +
+                `${stuckChild.command.slice(0, 80)} — sending Ctrl+C`);
+            this.sessionManager.sendKey(tmuxSession, 'C-c');
+            this.recordIntervention(tmuxSession, EscalationLevel.CtrlC, 'Sent Ctrl+C', stuckChild);
+        }
+        else if (existing) {
+            this.escalationState.delete(tmuxSession);
+        }
+    }
+    handleEscalation(tmuxSession, state) {
+        const now = Date.now();
+        if (!this.isProcessAlive(state.stuckChildPid)) {
+            console.log(`[Watchdog] "${tmuxSession}": stuck process ${state.stuckChildPid} died — recovered`);
+            this.emit('recovery', tmuxSession, state.level);
+            this.escalationState.delete(tmuxSession);
+            return;
+        }
+        const timeInLevel = now - state.levelEnteredAt;
+        const nextLevel = state.level + 1;
+        if (nextLevel > EscalationLevel.KillSession) {
+            if (state.retryCount >= MAX_RETRIES) {
+                console.log(`[Watchdog] "${tmuxSession}": max retries reached — giving up`);
+                this.escalationState.delete(tmuxSession);
+                return;
+            }
+            state.level = EscalationLevel.CtrlC;
+            state.levelEnteredAt = now;
+            state.retryCount++;
+            this.sessionManager.sendKey(tmuxSession, 'C-c');
+            this.recordIntervention(tmuxSession, EscalationLevel.CtrlC, `Retry ${state.retryCount}: Sent Ctrl+C`, {
+                pid: state.stuckChildPid, command: state.stuckCommand, elapsedMs: 0,
+            });
+            return;
+        }
+        const delayForNext = ESCALATION_DELAYS[nextLevel] ?? 15_000;
+        if (timeInLevel < delayForNext)
+            return;
+        state.level = nextLevel;
+        state.levelEnteredAt = now;
+        const child = { pid: state.stuckChildPid, command: state.stuckCommand, elapsedMs: 0 };
+        switch (state.level) {
+            case EscalationLevel.SigTerm:
+                console.log(`[Watchdog] "${tmuxSession}": sending SIGTERM to ${state.stuckChildPid}`);
+                this.sendSignal(state.stuckChildPid, 'SIGTERM');
+                this.recordIntervention(tmuxSession, EscalationLevel.SigTerm, `SIGTERM ${state.stuckChildPid}`, child);
+                break;
+            case EscalationLevel.SigKill:
+                console.log(`[Watchdog] "${tmuxSession}": sending SIGKILL to ${state.stuckChildPid}`);
+                this.sendSignal(state.stuckChildPid, 'SIGKILL');
+                this.recordIntervention(tmuxSession, EscalationLevel.SigKill, `SIGKILL ${state.stuckChildPid}`, child);
+                break;
+            case EscalationLevel.KillSession:
+                console.log(`[Watchdog] "${tmuxSession}": killing tmux session`);
+                this.killTmuxSession(tmuxSession);
+                this.recordIntervention(tmuxSession, EscalationLevel.KillSession, 'Killed tmux session', child);
+                this.escalationState.delete(tmuxSession);
+                break;
+        }
+    }
+    // --- Process utilities (self-contained, no shared module) ---
+    getClaudePid(tmuxSession) {
+        try {
+            // Get pane PID
+            const panePidStr = execSync(`${this.config.sessions.tmuxPath} list-panes -t "=${tmuxSession}" -F "#{pane_pid}" 2>/dev/null`, { encoding: 'utf-8', timeout: 5000 }).trim().split('\n')[0];
+            if (!panePidStr)
+                return null;
+            const panePid = parseInt(panePidStr, 10);
+            if (isNaN(panePid))
+                return null;
+            // Find claude child
+            const claudePidStr = execSync(`pgrep -P ${panePid} -f claude 2>/dev/null | head -1`, { encoding: 'utf-8', timeout: 5000 }).trim();
+            if (!claudePidStr)
+                return null;
+            const pid = parseInt(claudePidStr, 10);
+            return isNaN(pid) ? null : pid;
+        }
+        catch {
+            return null;
+        }
+    }
+    getChildProcesses(pid) {
+        try {
+            const childPidsStr = execSync(`pgrep -P ${pid} 2>/dev/null`, { encoding: 'utf-8', timeout: 5000 }).trim();
+            if (!childPidsStr)
+                return [];
+            const childPids = childPidsStr.split('\n').filter(Boolean).join(',');
+            if (!childPids)
+                return [];
+            const output = execSync(`ps -o pid=,etime=,command= -p ${childPids} 2>/dev/null`, { encoding: 'utf-8', timeout: 5000 }).trim();
+            if (!output)
+                return [];
+            const results = [];
+            for (const line of output.split('\n')) {
+                const match = line.trim().match(/^(\d+)\s+([\d:.-]+)\s+(.+)$/);
+                if (!match)
+                    continue;
+                const childPid = parseInt(match[1], 10);
+                if (isNaN(childPid))
+                    continue;
+                results.push({
+                    pid: childPid,
+                    command: match[3],
+                    elapsedMs: this.parseElapsed(match[2]),
+                });
+            }
+            return results;
+        }
+        catch {
+            return [];
+        }
+    }
+    isExcluded(command) {
+        for (const pattern of EXCLUDED_PATTERNS) {
+            if (command.includes(pattern))
+                return true;
+        }
+        for (const prefix of EXCLUDED_PREFIXES) {
+            if (command.startsWith(prefix))
+                return true;
+        }
+        return false;
+    }
+    parseElapsed(elapsed) {
+        let days = 0;
+        let timePart = elapsed;
+        if (elapsed.includes('-')) {
+            const [d, t] = elapsed.split('-');
+            days = parseInt(d, 10);
+            timePart = t;
+        }
+        const parts = timePart.split(':').map(Number);
+        let seconds = 0;
+        if (parts.length === 3)
+            seconds = parts[0] * 3600 + parts[1] * 60 + parts[2];
+        else if (parts.length === 2)
+            seconds = parts[0] * 60 + parts[1];
+        else
+            seconds = parts[0];
+        return (days * 86400 + seconds) * 1000;
+    }
+    sendSignal(pid, signal) {
+        try {
+            process.kill(pid, signal);
+        }
+        catch (err) {
+            if (err.code !== 'ESRCH') {
+                console.error(`[Watchdog] Failed to send ${signal} to ${pid}:`, err);
+            }
+        }
+    }
+    isProcessAlive(pid) {
+        try {
+            process.kill(pid, 0);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    killTmuxSession(tmuxSession) {
+        try {
+            execSync(`${this.config.sessions.tmuxPath} kill-session -t "=${tmuxSession}" 2>/dev/null`, { timeout: 5000, stdio: 'ignore' });
+        }
+        catch { }
+    }
+    recordIntervention(sessionName, level, action, child) {
+        const event = {
+            sessionName,
+            level,
+            action,
+            stuckCommand: child.command.slice(0, 200),
+            stuckPid: child.pid,
+            timestamp: Date.now(),
+        };
+        this.interventionHistory.push(event);
+        if (this.interventionHistory.length > 50) {
+            this.interventionHistory = this.interventionHistory.slice(-50);
+        }
+        this.emit('intervention', event);
+    }
+}
+//# sourceMappingURL=SessionWatchdog.js.map

package/dist/server/AgentServer.d.ts

@@ -21,6 +21,7 @@ import type { TelegraphService } from '../publishing/TelegraphService.js';
 import type { PrivateViewer } from '../publishing/PrivateViewer.js';
 import type { TunnelManager } from '../tunnel/TunnelManager.js';
 import type { EvolutionManager } from '../core/EvolutionManager.js';
+import type { SessionWatchdog } from '../monitoring/SessionWatchdog.js';
 export declare class AgentServer {
     private app;
     private server;
@@ -43,6 +44,7 @@ export declare class AgentServer {
         viewer?: PrivateViewer;
         tunnel?: TunnelManager;
         evolution?: EvolutionManager;
+        watchdog?: SessionWatchdog;
     });
     /**
      * Start the HTTP server.

package/dist/server/AgentServer.js

@@ -39,6 +39,7 @@ export class AgentServer {
             viewer: options.viewer ?? null,
             tunnel: options.tunnel ?? null,
             evolution: options.evolution ?? null,
+            watchdog: options.watchdog ?? null,
             startTime: this.startTime,
         });
         this.app.use(routes);

package/dist/server/WebSocketManager.d.ts

@@ -0,0 +1,62 @@
+/**
+ * WebSocket Manager — real-time terminal streaming for the dashboard.
+ *
+ * Handles client subscriptions to tmux sessions, streams terminal output
+ * via diff-based updates, and forwards input to sessions.
+ *
+ * Protocol (JSON messages):
+ *
+ * Client → Server:
+ *   { type: 'subscribe', session: 'session-name' }
+ *   { type: 'unsubscribe', session: 'session-name' }
+ *   { type: 'input', session: 'session-name', text: 'some input' }
+ *   { type: 'key', session: 'session-name', key: 'C-c' }
+ *   { type: 'ping' }
+ *
+ * Server → Client:
+ *   { type: 'output', session: 'session-name', data: '...terminal output...' }
+ *   { type: 'sessions', sessions: [...] }
+ *   { type: 'session_ended', session: 'session-name' }
+ *   { type: 'subscribed', session: 'session-name' }
+ *   { type: 'unsubscribed', session: 'session-name' }
+ *   { type: 'input_ack', session: 'session-name', success: true }
+ *   { type: 'pong' }
+ *   { type: 'error', message: '...' }
+ */
+import type { Server as HttpServer } from 'node:http';
+import type { SessionManager } from '../core/SessionManager.js';
+import type { StateManager } from '../core/StateManager.js';
+export declare class WebSocketManager {
+    private wss;
+    private clients;
+    private sessionOutputCache;
+    private streamInterval;
+    private heartbeatInterval;
+    private sessionBroadcastInterval;
+    private sessionManager;
+    private state;
+    private authToken?;
+    constructor(options: {
+        server: HttpServer;
+        sessionManager: SessionManager;
+        state: StateManager;
+        authToken?: string;
+    });
+    private authenticate;
+    private verifyToken;
+    private handleMessage;
+    /**
+     * Stream terminal output to subscribed clients.
+     * Uses diff-based approach: only sends new content since last capture.
+     */
+    private startStreaming;
+    private sendSessionList;
+    private broadcastSessionList;
+    private clientId;
+    private send;
+    /**
+     * Graceful shutdown — close all connections and stop intervals.
+     */
+    shutdown(): void;
+}
+//# sourceMappingURL=WebSocketManager.d.ts.map