@proletariat/cli 0.3.96 → 0.3.97

package/dist/lib/session/tmux-watchdog.js

@@ -0,0 +1,424 @@
+/**
+ * Tmux Watchdog
+ *
+ * Detects tmux server crashes and attempts automatic recovery of agent sessions.
+ *
+ * Problem: When the tmux server crashes (macOS sleep, resource pressure, OOM),
+ * ALL agent sessions are silently lost. The existing heartbeat system treats
+ * each agent as individually stale, missing the root cause. There is no
+ * recovery or user notification.
+ *
+ * Solution: This watchdog monitors the tmux server process itself. When a
+ * crash is detected (server transitions from alive→dead while agents were
+ * active), it:
+ *
+ * 1. Marks all affected executions with a crash-specific error
+ * 2. Attempts to re-create tmux sessions and restart agents
+ * 3. Logs crash events for user notification
+ *
+ * Architecture:
+ * - Runs as Phase 0 in SessionWatcher.runCycle(), before heartbeat checks
+ * - Maintains last-known server state per environment (host + each container)
+ * - Uses session-utils.ts for server status checks
+ * - Uses session restart logic to re-launch agents in recovered sessions
+ */
+import { execSync, execFileSync } from 'node:child_process';
+import { getHostTmuxServerStatus, getContainerTmuxServerStatus, } from '../execution/session-utils.js';
+// =============================================================================
+// Tmux Watchdog
+// =============================================================================
+export class TmuxWatchdog {
+    storage;
+    autoRecover;
+    log;
+    onCrashDetected;
+    /**
+     * Tracks last known tmux server state.
+     * Key: 'host' for host server, containerId for container servers.
+     * Value: true = server was alive, false = server was down.
+     */
+    lastServerState = new Map();
+    /**
+     * Tracks which executions were active when we last saw the server alive.
+     * Key: 'host' or containerId.
+     * Value: array of execution IDs that were active in that environment.
+     */
+    activeExecutionsSnapshot = new Map();
+    /**
+     * Tracks crash events we've already handled to avoid duplicate recovery.
+     * Key: 'host' or containerId.
+     * Value: timestamp of when we detected the crash.
+     */
+    handledCrashes = new Map();
+    constructor(options) {
+        this.storage = options.storage;
+        this.autoRecover = options.autoRecover ?? true;
+        this.log = options.log ?? (() => { });
+        this.onCrashDetected = options.onCrashDetected;
+    }
+    /**
+     * Run a single watchdog check cycle.
+     *
+     * Checks tmux server health for host and all containers with active executions.
+     * Detects crashes (server alive→dead transition) and attempts recovery.
+     */
+    async checkAndRecover() {
+        const result = {
+            hostServerAlive: true,
+            containerServersAlive: new Map(),
+            crashEvents: [],
+            totalAffected: 0,
+            totalRecovered: 0,
+        };
+        // Get all active executions
+        const runningExecs = this.storage.listExecutions({ status: 'running' });
+        const startingExecs = this.storage.listExecutions({ status: 'starting' });
+        const activeExecs = [...runningExecs, ...startingExecs];
+        // Group by environment
+        const hostExecs = activeExecs.filter(e => e.environment === 'host' || e.environment === 'sandbox');
+        const containerExecs = activeExecs.filter(e => (e.environment === 'devcontainer' || e.environment === 'docker') && e.containerId);
+        // Group container executions by container ID
+        const containerGroups = new Map();
+        for (const exec of containerExecs) {
+            const cId = exec.containerId;
+            const group = containerGroups.get(cId) || [];
+            group.push(exec);
+            containerGroups.set(cId, group);
+        }
+        // === Check host tmux server ===
+        if (hostExecs.length > 0 || this.lastServerState.has('host')) {
+            const hostStatus = getHostTmuxServerStatus();
+            const hostAlive = hostStatus === 'running';
+            result.hostServerAlive = hostAlive;
+            const wasAlive = this.lastServerState.get('host');
+            // Update snapshot of active host executions when server is alive
+            if (hostAlive) {
+                this.activeExecutionsSnapshot.set('host', hostExecs.map(e => e.id));
+                // Clear any previous crash handling for host
+                this.handledCrashes.delete('host');
+            }
+            // Detect crash: server was alive, now it's dead, and there were active sessions
+            if (wasAlive === true && !hostAlive && !this.handledCrashes.has('host')) {
+                const snapshotIds = this.activeExecutionsSnapshot.get('host') || [];
+                // Get the actual execution objects for affected sessions
+                const affected = snapshotIds
+                    .map(id => this.storage.getExecution(id))
+                    .filter((e) => e !== null && (e.status === 'running' || e.status === 'starting'));
+                if (affected.length > 0) {
+                    const event = await this.handleCrash('host', undefined, affected);
+                    result.crashEvents.push(event);
+                    result.totalAffected += event.affectedExecutions.length;
+                    result.totalRecovered += event.recoveredSessions.length;
+                    this.handledCrashes.set('host', Date.now());
+                }
+            }
+            this.lastServerState.set('host', hostAlive);
+        }
+        // === Check container tmux servers ===
+        for (const [containerId, execs] of containerGroups) {
+            const containerStatus = getContainerTmuxServerStatus(containerId);
+            const containerAlive = containerStatus === 'running';
+            result.containerServersAlive.set(containerId, containerAlive);
+            const wasAlive = this.lastServerState.get(containerId);
+            if (containerAlive) {
+                this.activeExecutionsSnapshot.set(containerId, execs.map(e => e.id));
+                this.handledCrashes.delete(containerId);
+            }
+            if (wasAlive === true && !containerAlive && !this.handledCrashes.has(containerId)) {
+                const snapshotIds = this.activeExecutionsSnapshot.get(containerId) || [];
+                const affected = snapshotIds
+                    .map(id => this.storage.getExecution(id))
+                    .filter((e) => e !== null && (e.status === 'running' || e.status === 'starting'));
+                if (affected.length > 0) {
+                    const event = await this.handleCrash('container', containerId, affected);
+                    result.crashEvents.push(event);
+                    result.totalAffected += event.affectedExecutions.length;
+                    result.totalRecovered += event.recoveredSessions.length;
+                    this.handledCrashes.set(containerId, Date.now());
+                }
+            }
+            this.lastServerState.set(containerId, containerAlive);
+        }
+        // Clean up tracking for containers that no longer have active executions
+        for (const key of this.lastServerState.keys()) {
+            if (key !== 'host' && !containerGroups.has(key)) {
+                this.lastServerState.delete(key);
+                this.activeExecutionsSnapshot.delete(key);
+                this.handledCrashes.delete(key);
+            }
+        }
+        return result;
+    }
+    /**
+     * Handle a detected tmux server crash.
+     * Marks affected executions and attempts recovery.
+     */
+    async handleCrash(environment, containerId, affected) {
+        const envLabel = environment === 'host' ? 'host' : `container ${containerId?.slice(0, 12)}`;
+        this.log(`[watchdog] TMUX SERVER CRASH detected on ${envLabel} — ${affected.length} agent(s) affected`);
+        const event = {
+            timestamp: new Date(),
+            environment,
+            containerId,
+            affectedExecutions: affected,
+            recoveryAttempted: false,
+            recoveredSessions: [],
+            failedRecoveries: [],
+        };
+        // Mark affected executions with crash-specific error
+        for (const exec of affected) {
+            this.log(`[watchdog]   Affected: ${exec.agentName} (${exec.ticketId}) — session: ${exec.sessionId || 'unknown'}`);
+            this.storage.updateStatus(exec.id, 'failed', undefined, `tmux server crash on ${envLabel} — all sessions lost. ` +
+                `${this.autoRecover ? 'Auto-recovery attempted.' : 'Manual restart required.'}`);
+            this.storage.updateLifecycleState(exec.id, 'died');
+        }
+        // Attempt auto-recovery if enabled
+        if (this.autoRecover) {
+            event.recoveryAttempted = true;
+            await this.attemptRecovery(event);
+        }
+        // Fire callback
+        if (this.onCrashDetected) {
+            await this.onCrashDetected(event);
+        }
+        // Log recovery summary
+        if (event.recoveryAttempted) {
+            if (event.recoveredSessions.length > 0) {
+                this.log(`[watchdog] Recovery: ${event.recoveredSessions.length}/${affected.length} sessions restored`);
+            }
+            if (event.failedRecoveries.length > 0) {
+                this.log(`[watchdog] Recovery failed for: ${event.failedRecoveries.join(', ')}`);
+            }
+        }
+        return event;
+    }
+    /**
+     * Attempt to recover agent sessions after a tmux server crash.
+     *
+     * Recovery steps:
+     * 1. Wait briefly for tmux server to come back (macOS wake, Docker resume)
+     * 2. Create new tmux sessions with the original session names
+     * 3. Navigate to the working directory
+     * 4. Re-launch Claude Code with --resume flag
+     */
+    async attemptRecovery(event) {
+        const { environment, containerId, affectedExecutions } = event;
+        // Wait briefly for tmux server to recover (macOS wake scenario)
+        const serverBack = await this.waitForServerRecovery(environment, containerId, 5000);
+        if (!serverBack) {
+            // For host: try starting a new tmux server (it auto-starts on new-session)
+            // For container: tmux server restarts on next tmux command
+            this.log('[watchdog] Tmux server did not auto-recover, will try to start new sessions');
+        }
+        for (const exec of affectedExecutions) {
+            const sessionName = exec.sessionId;
+            if (!sessionName) {
+                event.failedRecoveries.push(exec.agentName);
+                this.log(`[watchdog]   Skip ${exec.agentName}: no session ID recorded`);
+                continue;
+            }
+            try {
+                const recovered = this.restartSession(exec, environment, containerId);
+                if (recovered) {
+                    event.recoveredSessions.push(exec.agentName);
+                    // Update execution back to running
+                    this.storage.updateStatus(exec.id, 'running');
+                    this.storage.updateLifecycleState(exec.id, 'healthy');
+                    this.storage.updateHeartbeat(exec.id);
+                    this.log(`[watchdog]   Recovered: ${exec.agentName} (${exec.ticketId})`);
+                }
+                else {
+                    event.failedRecoveries.push(exec.agentName);
+                    this.log(`[watchdog]   Failed to recover: ${exec.agentName} (${exec.ticketId})`);
+                }
+            }
+            catch (error) {
+                event.failedRecoveries.push(exec.agentName);
+                this.log(`[watchdog]   Error recovering ${exec.agentName}: ${error instanceof Error ? error.message : error}`);
+            }
+        }
+    }
+    /**
+     * Wait for the tmux server to come back online.
+     * Returns true if the server recovered within the timeout.
+     */
+    async waitForServerRecovery(environment, containerId, timeoutMs) {
+        const start = Date.now();
+        const checkInterval = 1000;
+        while (Date.now() - start < timeoutMs) {
+            const status = environment === 'host'
+                ? getHostTmuxServerStatus()
+                : getContainerTmuxServerStatus(containerId);
+            if (status === 'running') {
+                this.log('[watchdog] Tmux server recovered');
+                return true;
+            }
+            await new Promise(resolve => setTimeout(resolve, checkInterval));
+        }
+        return false;
+    }
+    /**
+     * Restart an agent session by creating a new tmux session and re-launching Claude Code.
+     */
+    restartSession(exec, environment, containerId) {
+        const sessionName = exec.sessionId;
+        // Build the claude command for re-launch
+        let claudeCmd = 'claude --resume';
+        if (exec.permissionMode === 'danger') {
+            claudeCmd += ' --dangerously-skip-permissions';
+        }
+        if (environment === 'container' && containerId) {
+            return this.restartContainerSession(containerId, sessionName, claudeCmd);
+        }
+        return this.restartHostSession(sessionName, claudeCmd);
+    }
+    /**
+     * Restart a host tmux session.
+     */
+    restartHostSession(sessionName, command) {
+        try {
+            // Create new tmux session — this also starts the server if it's not running
+            execFileSync('tmux', [
+                'new-session', '-d',
+                '-s', sessionName,
+                '-n', sessionName,
+                command,
+            ], {
+                stdio: ['pipe', 'pipe', 'pipe'],
+                timeout: 10000,
+            });
+            // Enable mouse support
+            try {
+                execFileSync('tmux', ['set-option', '-g', 'mouse', 'on'], {
+                    stdio: ['pipe', 'pipe', 'pipe'],
+                    timeout: 5000,
+                });
+            }
+            catch {
+                // Non-critical
+            }
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Restart a tmux session inside a Docker container.
+     */
+    restartContainerSession(containerId, sessionName, command) {
+        try {
+            // Create new tmux session inside container
+            execFileSync('docker', [
+                'exec', containerId,
+                'tmux', 'new-session', '-d',
+                '-s', sessionName,
+                '-n', sessionName,
+                command,
+            ], {
+                stdio: ['pipe', 'pipe', 'pipe'],
+                timeout: 15000,
+            });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Get the current known state of servers.
+     * Useful for diagnostics and testing.
+     */
+    getServerStates() {
+        return new Map(this.lastServerState);
+    }
+    /**
+     * Reset internal state. Useful for testing.
+     */
+    reset() {
+        this.lastServerState.clear();
+        this.activeExecutionsSnapshot.clear();
+        this.handledCrashes.clear();
+    }
+    /**
+     * Seed the initial server state without triggering crash detection.
+     * Call this once during watcher startup to establish baseline.
+     */
+    seedServerState(activeExecs) {
+        // Group by environment
+        const hostExecs = activeExecs.filter(e => e.environment === 'host' || e.environment === 'sandbox');
+        const containerExecs = activeExecs.filter(e => (e.environment === 'devcontainer' || e.environment === 'docker') && e.containerId);
+        if (hostExecs.length > 0) {
+            const hostStatus = getHostTmuxServerStatus();
+            this.lastServerState.set('host', hostStatus === 'running');
+            this.activeExecutionsSnapshot.set('host', hostExecs.map(e => e.id));
+        }
+        const containerGroups = new Map();
+        for (const exec of containerExecs) {
+            const cId = exec.containerId;
+            const group = containerGroups.get(cId) || [];
+            group.push(exec);
+            containerGroups.set(cId, group);
+        }
+        for (const [cId, execs] of containerGroups) {
+            const status = getContainerTmuxServerStatus(cId);
+            this.lastServerState.set(cId, status === 'running');
+            this.activeExecutionsSnapshot.set(cId, execs.map(e => e.id));
+        }
+    }
+}
+// =============================================================================
+// Notification Helpers
+// =============================================================================
+/**
+ * Format a crash event into a human-readable notification string.
+ */
+export function formatCrashNotification(event) {
+    const envLabel = event.environment === 'host'
+        ? 'host tmux server'
+        : `container ${event.containerId?.slice(0, 12)} tmux server`;
+    const lines = [
+        `⚠️  TMUX SERVER CRASH — ${envLabel}`,
+        `   Time: ${event.timestamp.toLocaleString()}`,
+        `   Affected agents: ${event.affectedExecutions.length}`,
+    ];
+    for (const exec of event.affectedExecutions) {
+        lines.push(`     • ${exec.agentName} (${exec.ticketId})`);
+    }
+    if (event.recoveryAttempted) {
+        if (event.recoveredSessions.length === event.affectedExecutions.length) {
+            lines.push(`   Recovery: ALL ${event.recoveredSessions.length} sessions restored`);
+        }
+        else if (event.recoveredSessions.length > 0) {
+            lines.push(`   Recovery: ${event.recoveredSessions.length}/${event.affectedExecutions.length} restored`);
+            lines.push(`   Failed: ${event.failedRecoveries.join(', ')}`);
+        }
+        else {
+            lines.push(`   Recovery: FAILED — all ${event.affectedExecutions.length} sessions need manual restart`);
+            lines.push('   Run: prlt session health');
+        }
+    }
+    else {
+        lines.push('   Auto-recovery: disabled');
+        lines.push('   Run: prlt session health');
+    }
+    return lines.join('\n');
+}
+/**
+ * Send a desktop notification about a tmux crash (best effort).
+ * Uses osascript on macOS, notify-send on Linux.
+ */
+export function sendDesktopNotification(title, message) {
+    try {
+        if (process.platform === 'darwin') {
+            execSync(`osascript -e 'display notification "${message.replace(/"/g, '\\"')}" with title "${title.replace(/"/g, '\\"')}"'`, { stdio: 'pipe', timeout: 5000 });
+        }
+        else if (process.platform === 'linux') {
+            execSync(`notify-send "${title.replace(/"/g, '\\"')}" "${message.replace(/"/g, '\\"')}" 2>/dev/null`, { stdio: 'pipe', timeout: 5000 });
+        }
+    }
+    catch {
+        // Best effort — notification systems may not be available
+    }
+}
+//# sourceMappingURL=tmux-watchdog.js.map

package/dist/lib/session/tmux-watchdog.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tmux-watchdog.js","sourceRoot":"","sources":["../../../src/lib/session/tmux-watchdog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAG3D,OAAO,EACL,uBAAuB,EACvB,4BAA4B,GAE7B,MAAM,+BAA+B,CAAA;AAqDtC,gFAAgF;AAChF,gBAAgB;AAChB,gFAAgF;AAEhF,MAAM,OAAO,YAAY;IACf,OAAO,CAAkB;IACzB,WAAW,CAAS;IACpB,GAAG,CAAuB;IAC1B,eAAe,CAAkD;IAEzE;;;;OAIG;IACK,eAAe,GAAG,IAAI,GAAG,EAAmB,CAAA;IAEpD;;;;OAIG;IACK,wBAAwB,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE9D;;;;OAIG;IACK,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAA;IAElD,YAAY,OAA4B;QACtC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAA;QAC9B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAA;QAC9C,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAA;IAChD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,eAAe;QACnB,MAAM,MAAM,GAAwB;YAClC,eAAe,EAAE,IAAI;YACrB,qBAAqB,EAAE,IAAI,GAAG,EAAE;YAChC,WAAW,EAAE,EAAE;YACf,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;SAClB,CAAA;QAED,4BAA4B;QAC5B,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;QACvE,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;QACzE,MAAM,WAAW,GAAG,CAAC,GAAG,YAAY,EAAE,GAAG,aAAa,CAAC,CAAA;QAEvD,uBAAuB;QACvB,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAClC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,MAAM,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS,CAC7D,CAAA;QACD,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,KAAK,cAAc,IAAI,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,CACvF,CAAA;QAED,6CAA6C;QAC7C,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAA;QACtD,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAY,CAAA;YAC7B,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;YAC5C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChB,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACjC,CAAC;QAED,iCAAiC;QACjC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7D,MAAM,UAAU,GAAG,uBAAuB,EAAE,CAAA;YAC5C,MAAM,SAAS,GAAG,UAAU,KAAK,SAAS,CAAA;YAC1C,MAAM,CAAC,eAAe,GAAG,SAAS,CAAA;YAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAEjD,iEAAiE;YACjE,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;gBACnE,6CAA6C;gBAC7C,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YACpC,CAAC;YAED,gFAAgF;YAChF,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxE,MAAM,WAAW,GAAG,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;gBACnE,yDAAyD;gBACzD,MAAM,QAAQ,GAAG,WAAW;qBACzB,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;qBACxC,MAAM,CAAC,CAAC,CAAC,EAAkB,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAA;gBAEnG,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAA;oBACjE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAC9B,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAA;oBACvD,MAAM,CAAC,cAAc,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAA;oBACvD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACH,CAAC;YAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAC7C,CAAC;QAED,uCAAuC;QACvC,KAAK,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,eAAe,EAAE,CAAC;YACnD,MAAM,eAAe,GAAG,4BAA4B,CAAC,WAAW,CAAC,CAAA;YACjE,MAAM,cAAc,GAAG,eAAe,KAAK,SAAS,CAAA;YACpD,MAAM,CAAC,qBAAqB,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAA;YAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;YAEtD,IAAI,cAAc,EAAE,CAAC;gBACnB,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;gBACpE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YACzC,CAAC;YAED,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClF,MAAM,WAAW,GAAG,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAA;gBACxE,MAAM,QAAQ,GAAG,WAAW;qBACzB,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;qBACxC,MAAM,CAAC,CAAC,CAAC,EAAkB,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAA;gBAEnG,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAA;oBACxE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAC9B,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAA;oBACvD,MAAM,CAAC,cAAc,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAA;oBACvD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;gBAClD,CAAC;YACH,CAAC;YAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAA;QACvD,CAAC;QAED,yEAAyE;QACzE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9C,IAAI,GAAG,KAAK,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAChC,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBACzC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACjC,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,WAAW,CACvB,WAAiC,EACjC,WAA+B,EAC/B,QAAqB;QAErB,MAAM,QAAQ,GAAG,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,WAAW,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAA;QAC3F,IAAI,CAAC,GAAG,CACN,4CAA4C,QAAQ,MAAM,QAAQ,CAAC,MAAM,oBAAoB,CAC9F,CAAA;QAED,MAAM,KAAK,GAAmB;YAC5B,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,WAAW;YACX,WAAW;YACX,kBAAkB,EAAE,QAAQ;YAC5B,iBAAiB,EAAE,KAAK;YACxB,iBAAiB,EAAE,EAAE;YACrB,gBAAgB,EAAE,EAAE;SACrB,CAAA;QAED,qDAAqD;QACrD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,GAAG,CACN,0BAA0B,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,QAAQ,gBAAgB,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE,CACxG,CAAA;YACD,IAAI,CAAC,OAAO,CAAC,YAAY,CACvB,IAAI,CAAC,EAAE,EACP,QAAQ,EACR,SAAS,EACT,wBAAwB,QAAQ,wBAAwB;gBACxD,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,0BAA0B,EAAE,CAChF,CAAA;YACD,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;QACpD,CAAC;QAED,mCAAmC;QACnC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,KAAK,CAAC,iBAAiB,GAAG,IAAI,CAAA;YAC9B,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC;QAED,gBAAgB;QAChB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC;QAED,uBAAuB;QACvB,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,IAAI,CAAC,GAAG,CACN,wBAAwB,KAAK,CAAC,iBAAiB,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,oBAAoB,CAC9F,CAAA;YACH,CAAC;YACD,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,IAAI,CAAC,GAAG,CACN,mCAAmC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACvE,CAAA;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,eAAe,CAAC,KAAqB;QACjD,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,kBAAkB,EAAE,GAAG,KAAK,CAAA;QAE9D,gEAAgE;QAChE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,CAAC,CAAA;QAEnF,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,2EAA2E;YAC3E,2DAA2D;YAC3D,IAAI,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAA;QACzF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAC;YACtC,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAA;YAClC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBAC3C,IAAI,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,SAAS,0BAA0B,CAAC,CAAA;gBACvE,SAAQ;YACV,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,WAAW,EAAE,WAAW,CAAC,CAAA;gBACrE,IAAI,SAAS,EAAE,CAAC;oBACd,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;oBAE5C,mCAAmC;oBACnC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;oBAC7C,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;oBACrD,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;oBAErC,IAAI,CAAC,GAAG,CAAC,2BAA2B,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAA;gBAC1E,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;oBAC3C,IAAI,CAAC,GAAG,CAAC,mCAAmC,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAA;gBAClF,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBAC3C,IAAI,CAAC,GAAG,CACN,iCAAiC,IAAI,CAAC,SAAS,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,CACrG,CAAA;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,qBAAqB,CACjC,WAAiC,EACjC,WAA+B,EAC/B,SAAiB;QAEjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,MAAM,aAAa,GAAG,IAAI,CAAA;QAE1B,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;YACtC,MAAM,MAAM,GACV,WAAW,KAAK,MAAM;gBACpB,CAAC,CAAC,uBAAuB,EAAE;gBAC3B,CAAC,CAAC,4BAA4B,CAAC,WAAY,CAAC,CAAA;YAEhD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAA;gBAC5C,OAAO,IAAI,CAAA;YACb,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAA;QAClE,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;OAEG;IACK,cAAc,CACpB,IAAe,EACf,WAAiC,EACjC,WAA+B;QAE/B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAU,CAAA;QAEnC,yCAAyC;QACzC,IAAI,SAAS,GAAG,iBAAiB,CAAA;QACjC,IAAI,IAAI,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;YACrC,SAAS,IAAI,iCAAiC,CAAA;QAChD,CAAC;QAED,IAAI,WAAW,KAAK,WAAW,IAAI,WAAW,EAAE,CAAC;YAC/C,OAAO,IAAI,CAAC,uBAAuB,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAA;QAC1E,CAAC;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;IACxD,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,WAAmB,EAAE,OAAe;QAC7D,IAAI,CAAC;YACH,4EAA4E;YAC5E,YAAY,CAAC,MAAM,EAAE;gBACnB,aAAa,EAAE,IAAI;gBACnB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,WAAW;gBACjB,OAAO;aACR,EAAE;gBACD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;gBAC/B,OAAO,EAAE,KAAK;aACf,CAAC,CAAA;YAEF,uBAAuB;YACvB,IAAI,CAAC;gBACH,YAAY,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE;oBACxD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC/B,OAAO,EAAE,IAAI;iBACd,CAAC,CAAA;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,eAAe;YACjB,CAAC;YAED,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,uBAAuB,CAC7B,WAAmB,EACnB,WAAmB,EACnB,OAAe;QAEf,IAAI,CAAC;YACH,2CAA2C;YAC3C,YAAY,CAAC,QAAQ,EAAE;gBACrB,MAAM,EAAE,WAAW;gBACnB,MAAM,EAAE,aAAa,EAAE,IAAI;gBAC3B,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,WAAW;gBACjB,OAAO;aACR,EAAE;gBACD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;gBAC/B,OAAO,EAAE,KAAK;aACf,CAAC,CAAA;YACF,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;QAC5B,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,CAAA;QACrC,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,WAAwB;QACtC,uBAAuB;QACvB,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAClC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,MAAM,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS,CAC7D,CAAA;QACD,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,KAAK,cAAc,IAAI,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,CACvF,CAAA;QAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,uBAAuB,EAAE,CAAA;YAC5C,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,KAAK,SAAS,CAAC,CAAA;YAC1D,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QACrE,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAA;QACtD,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAY,CAAA;YAC7B,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;YAC5C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChB,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACjC,CAAC;QAED,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,eAAe,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,4BAA4B,CAAC,GAAG,CAAC,CAAA;YAChD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,SAAS,CAAC,CAAA;YACnD,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAC9D,CAAC;IACH,CAAC;CACF;AAED,gFAAgF;AAChF,uBAAuB;AACvB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAqB;IAC3D,MAAM,QAAQ,GACZ,KAAK,CAAC,WAAW,KAAK,MAAM;QAC1B,CAAC,CAAC,kBAAkB;QACpB,CAAC,CAAC,aAAa,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAA;IAEhE,MAAM,KAAK,GAAa;QACtB,2BAA2B,QAAQ,EAAE;QACrC,YAAY,KAAK,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;QAC9C,uBAAuB,KAAK,CAAC,kBAAkB,CAAC,MAAM,EAAE;KACzD,CAAA;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAA;IAC3D,CAAC;IAED,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,KAAK,KAAK,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACvE,KAAK,CAAC,IAAI,CAAC,oBAAoB,KAAK,CAAC,iBAAiB,CAAC,MAAM,oBAAoB,CAAC,CAAA;QACpF,CAAC;aAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,KAAK,CAAC,IAAI,CACR,gBAAgB,KAAK,CAAC,iBAAiB,CAAC,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,MAAM,WAAW,CAC7F,CAAA;YACD,KAAK,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC/D,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,6BAA6B,KAAK,CAAC,kBAAkB,CAAC,MAAM,+BAA+B,CAAC,CAAA;YACvG,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAA;QACxC,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;IAC3C,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAa,EAAE,OAAe;IACpE,IAAI,CAAC;QACH,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAClC,QAAQ,CACN,uCAAuC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,iBAAiB,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAClH,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CACjC,CAAA;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACxC,QAAQ,CACN,gBAAgB,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,eAAe,EAC3F,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CACjC,CAAA;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,0DAA0D;IAC5D,CAAC;AACH,CAAC"}

package/dist/lib/session/watcher.d.ts

@@ -6,14 +6,15 @@
  * that can't self-report (OOM kills, zombie processes, container crashes).
  *
  * Architecture:
- * 1. Poll: Check all running executions for heartbeat timeout
- * 2. Update: Record heartbeats for alive agents (tmux pane inspection)
- * 3. Detect: Find stale executions that exceeded the timeout
- * 4. Act: Mark failed, kill containers, fire events
+ * Phase 0: Tmux server health check — detect full server crashes (PRLT-1115)
+ * Phase 1: Record heartbeats for alive agents (tmux pane inspection)
+ * Phase 2: Detect stale executions that exceeded the timeout
+ * Phase 3: Act on stale executions — mark failed, kill containers, fire events
  */
 import type Database from 'better-sqlite3';
 import type { AgentWork } from '../execution/types.js';
 import { type StaleExecution } from './heartbeat.js';
+import { TmuxWatchdog, type TmuxCrashEvent } from './tmux-watchdog.js';
 export interface WatcherOptions {
     /** Database connection */
     db: Database.Database;
@@ -23,10 +24,14 @@ export interface WatcherOptions {
     timeoutMinutes?: number;
     /** Whether to kill containers on timeout (default: true) */
     autoKill?: boolean;
+    /** Whether to auto-recover from tmux server crashes (default: true) */
+    autoRecover?: boolean;
     /** Logger function */
     log?: (msg: string) => void;
     /** Callback when a stale execution is detected and handled */
     onStaleDetected?: (execution: AgentWork, reason: string) => void | Promise<void>;
+    /** Callback when a tmux server crash is detected */
+    onCrashDetected?: (event: TmuxCrashEvent) => void | Promise<void>;
 }
 export interface WatchCycleResult {
     /** Number of active executions checked */
@@ -37,6 +42,10 @@ export interface WatchCycleResult {
     staleExecutions: StaleExecution[];
     /** Number of containers killed */
     containersKilled: number;
+    /** Tmux crash events detected in this cycle */
+    crashEvents: TmuxCrashEvent[];
+    /** Total sessions recovered from crashes */
+    crashRecoveries: number;
 }
 export declare class SessionWatcher {
     private storage;
@@ -44,10 +53,14 @@ export declare class SessionWatcher {
     private intervalMinutes;
     private timeoutMinutes;
     private autoKill;
+    private autoRecover;
     private log;
     private onStaleDetected?;
+    private onCrashDetected?;
     private timer;
     private running;
+    private watchdog;
+    private isFirstCycle;
     constructor(options: WatcherOptions);
     /**
      * Run a single watch cycle.
@@ -74,6 +87,11 @@ export declare class SessionWatcher {
         intervalMinutes: number;
         timeoutMinutes: number;
         autoKill: boolean;
+        autoRecover: boolean;
     };
+    /**
+     * Get the tmux watchdog instance (for testing/diagnostics).
+     */
+    getWatchdog(): TmuxWatchdog;
     private runCycleWithErrorHandling;
 }

package/dist/lib/session/watcher.js

@@ -6,13 +6,14 @@
  * that can't self-report (OOM kills, zombie processes, container crashes).
  *
  * Architecture:
- * 1. Poll: Check all running executions for heartbeat timeout
- * 2. Update: Record heartbeats for alive agents (tmux pane inspection)
- * 3. Detect: Find stale executions that exceeded the timeout
- * 4. Act: Mark failed, kill containers, fire events
+ * Phase 0: Tmux server health check — detect full server crashes (PRLT-1115)
+ * Phase 1: Record heartbeats for alive agents (tmux pane inspection)
+ * Phase 2: Detect stale executions that exceeded the timeout
+ * Phase 3: Act on stale executions — mark failed, kill containers, fire events
  */
 import { ExecutionStorage } from '../execution/storage.js';
 import { recordAllHeartbeats, detectStaleExecutions, killContainer, } from './heartbeat.js';
+import { TmuxWatchdog, formatCrashNotification, sendDesktopNotification, } from './tmux-watchdog.js';
 // =============================================================================
 // Session Watcher
 // =============================================================================
@@ -22,18 +23,40 @@ export class SessionWatcher {
     intervalMinutes;
     timeoutMinutes;
     autoKill;
+    autoRecover;
     log;
     onStaleDetected;
+    onCrashDetected;
     timer = null;
     running = false;
+    watchdog;
+    isFirstCycle = true;
     constructor(options) {
         this.db = options.db;
         this.storage = new ExecutionStorage(options.db);
         this.intervalMinutes = options.intervalMinutes ?? 5;
         this.timeoutMinutes = options.timeoutMinutes ?? 15;
         this.autoKill = options.autoKill ?? true;
+        this.autoRecover = options.autoRecover ?? true;
         this.log = options.log ?? (() => { });
         this.onStaleDetected = options.onStaleDetected;
+        this.onCrashDetected = options.onCrashDetected;
+        this.watchdog = new TmuxWatchdog({
+            storage: this.storage,
+            autoRecover: this.autoRecover,
+            log: this.log,
+            onCrashDetected: async (event) => {
+                // Send desktop notification
+                const notification = formatCrashNotification(event);
+                this.log(notification);
+                sendDesktopNotification('prlt: tmux server crash', `${event.affectedExecutions.length} agent(s) affected. ` +
+                    `${event.recoveredSessions.length} recovered.`);
+                // Fire user callback
+                if (this.onCrashDetected) {
+                    await this.onCrashDetected(event);
+                }
+            },
+        });
     }
     /**
      * Run a single watch cycle.
@@ -45,7 +68,25 @@ export class SessionWatcher {
             heartbeatsUpdated: 0,
             staleExecutions: [],
             containersKilled: 0,
+            crashEvents: [],
+            crashRecoveries: 0,
         };
+        // Phase 0: Tmux server health check (PRLT-1115)
+        // Detect full server crashes before doing individual heartbeat checks.
+        // On the first cycle, seed the watchdog state without triggering crash detection.
+        if (this.isFirstCycle) {
+            const activeExecs = [
+                ...this.storage.listExecutions({ status: 'running' }),
+                ...this.storage.listExecutions({ status: 'starting' }),
+            ];
+            this.watchdog.seedServerState(activeExecs);
+            this.isFirstCycle = false;
+        }
+        else {
+            const watchdogResult = await this.watchdog.checkAndRecover();
+            result.crashEvents = watchdogResult.crashEvents;
+            result.crashRecoveries = watchdogResult.totalRecovered;
+        }
         // Phase 1: Record heartbeats for all active executions
         // This is the "push" side — we check tmux panes and update heartbeats
         const heartbeats = recordAllHeartbeats(this.storage);
@@ -88,7 +129,8 @@ export class SessionWatcher {
         if (this.running)
             return;
         this.running = true;
-        this.log(`[watcher] Starting session watcher (interval: ${this.intervalMinutes}m, timeout: ${this.timeoutMinutes}m, auto-kill: ${this.autoKill})`);
+        this.log(`[watcher] Starting session watcher (interval: ${this.intervalMinutes}m, timeout: ${this.timeoutMinutes}m, ` +
+            `auto-kill: ${this.autoKill}, auto-recover: ${this.autoRecover})`);
         // Run initial cycle
         void this.runCycleWithErrorHandling();
         // Start polling
@@ -124,14 +166,30 @@ export class SessionWatcher {
             intervalMinutes: this.intervalMinutes,
             timeoutMinutes: this.timeoutMinutes,
             autoKill: this.autoKill,
+            autoRecover: this.autoRecover,
         };
     }
+    /**
+     * Get the tmux watchdog instance (for testing/diagnostics).
+     */
+    getWatchdog() {
+        return this.watchdog;
+    }
     async runCycleWithErrorHandling() {
         try {
             const result = await this.runCycle();
-            if (result.staleExecutions.length > 0) {
-                this.log(`[watcher] Cycle complete: ${result.checked} checked, ${result.heartbeatsUpdated} heartbeats, ` +
-                    `${result.staleExecutions.length} stale, ${result.containersKilled} killed`);
+            if (result.staleExecutions.length > 0 || result.crashEvents.length > 0) {
+                const parts = [
+                    `${result.checked} checked`,
+                    `${result.heartbeatsUpdated} heartbeats`,
+                    `${result.staleExecutions.length} stale`,
+                    `${result.containersKilled} killed`,
+                ];
+                if (result.crashEvents.length > 0) {
+                    parts.push(`${result.crashEvents.length} crash(es)`);
+                    parts.push(`${result.crashRecoveries} recovered`);
+                }
+                this.log(`[watcher] Cycle complete: ${parts.join(', ')}`);
             }
         }
         catch (error) {