agent-relay 1.6.0 → 2.0.0

package/packages/daemon/dist/orchestrator.js

@@ -0,0 +1,1143 @@
+/**
+ * Daemon Orchestrator
+ *
+ * Manages multiple workspace daemons and provides a unified API for the dashboard.
+ * This is the top-level service that runs by default, handling workspace switching
+ * and agent management across all connected repositories.
+ */
+import * as http from 'http';
+import * as path from 'path';
+import * as fs from 'fs';
+import { EventEmitter } from 'events';
+import { WebSocketServer, WebSocket } from 'ws';
+import { createLogger, metrics, getSupervisor, getMemoryMonitor, formatBytes, } from '@agent-relay/resiliency';
+import { Daemon } from './server.js';
+import { AgentSpawner } from '@agent-relay/bridge';
+import { getProjectPaths } from '@agent-relay/config';
+const logger = createLogger('orchestrator');
+function generateId() {
+    return Math.random().toString(36).substring(2, 15);
+}
+const DEFAULT_CONFIG = {
+    port: 3456,
+    host: 'localhost',
+    dataDir: path.join(process.env.HOME || '', '.agent-relay', 'orchestrator'),
+    autoStartDaemons: true,
+};
+const HEARTBEAT_INTERVAL_MS = 10_000;
+const RESOURCE_ALERT_COOLDOWN_MS = 60_000;
+const parsedCpuThreshold = parseFloat(process.env.AGENT_CPU_ALERT_THRESHOLD || '300');
+const CPU_ALERT_THRESHOLD = Number.isFinite(parsedCpuThreshold) ? parsedCpuThreshold : 300;
+export class Orchestrator extends EventEmitter {
+    config;
+    workspaces = new Map();
+    activeWorkspaceId;
+    server;
+    wss;
+    sessions = new Map();
+    supervisor = getSupervisor({
+        autoRestart: true,
+        maxRestarts: 5,
+        contextPersistence: { enabled: true, autoInjectOnRestart: true },
+    });
+    workspacesFile;
+    // Track alive status for ping/pong keepalive
+    clientAlive = new WeakMap();
+    pingInterval;
+    heartbeatInterval;
+    memoryMonitor = getMemoryMonitor({ checkIntervalMs: 10_000 });
+    agentHealth = new Map();
+    // Event handler references for cleanup
+    memorySampleHandler;
+    memoryAlertHandler;
+    constructor(config = {}) {
+        super();
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.workspacesFile = path.join(this.config.dataDir, 'workspaces.json');
+        // Ensure data directory exists
+        if (!fs.existsSync(this.config.dataDir)) {
+            fs.mkdirSync(this.config.dataDir, { recursive: true });
+        }
+        // Load existing workspaces
+        this.loadWorkspaces();
+    }
+    /**
+     * Start the orchestrator
+     */
+    async start() {
+        logger.info('Starting orchestrator', {
+            port: this.config.port,
+            host: this.config.host,
+        });
+        // Start supervisor
+        this.supervisor.start();
+        // Auto-start daemons for workspaces
+        if (this.config.autoStartDaemons) {
+            for (const [id, workspace] of this.workspaces) {
+                if (fs.existsSync(workspace.path)) {
+                    await this.startWorkspaceDaemon(id);
+                }
+            }
+        }
+        // Start HTTP server
+        this.server = http.createServer((req, res) => this.handleRequest(req, res));
+        // Setup WebSocket
+        this.wss = new WebSocketServer({ server: this.server });
+        this.wss.on('connection', (ws, req) => this.handleWebSocket(ws, req));
+        // Setup ping/pong keepalive (30 second interval)
+        this.pingInterval = setInterval(() => {
+            this.wss?.clients.forEach((ws) => {
+                if (this.clientAlive.get(ws) === false) {
+                    logger.info('WebSocket client unresponsive, closing');
+                    ws.terminate();
+                    return;
+                }
+                this.clientAlive.set(ws, false);
+                ws.ping();
+            });
+        }, 30000);
+        this.startHealthMonitoring();
+        return new Promise((resolve) => {
+            this.server.listen(this.config.port, this.config.host, () => {
+                logger.info('Orchestrator started', {
+                    url: `http://${this.config.host}:${this.config.port}`,
+                });
+                resolve();
+            });
+        });
+    }
+    /**
+     * Stop the orchestrator
+     */
+    async stop() {
+        logger.info('Stopping orchestrator');
+        // Clear ping interval
+        if (this.pingInterval) {
+            clearInterval(this.pingInterval);
+            this.pingInterval = undefined;
+        }
+        if (this.heartbeatInterval) {
+            clearInterval(this.heartbeatInterval);
+            this.heartbeatInterval = undefined;
+        }
+        // Clean up memory monitor event handlers before stopping
+        if (this.memorySampleHandler) {
+            this.memoryMonitor.off('sample', this.memorySampleHandler);
+            this.memorySampleHandler = undefined;
+        }
+        if (this.memoryAlertHandler) {
+            this.memoryMonitor.off('alert', this.memoryAlertHandler);
+            this.memoryAlertHandler = undefined;
+        }
+        this.memoryMonitor.stop();
+        // Stop all workspace daemons
+        for (const [id] of this.workspaces) {
+            await this.stopWorkspaceDaemon(id);
+        }
+        // Stop supervisor
+        this.supervisor.stop();
+        // Close WebSocket connections
+        if (this.wss) {
+            for (const ws of this.wss.clients) {
+                ws.close();
+            }
+            this.wss.close();
+        }
+        // Close HTTP server
+        if (this.server) {
+            return new Promise((resolve) => {
+                this.server.close(() => {
+                    logger.info('Orchestrator stopped');
+                    resolve();
+                });
+            });
+        }
+    }
+    // === Workspace Management ===
+    /**
+     * Add a workspace
+     */
+    addWorkspace(request) {
+        const resolvedPath = this.resolvePath(request.path);
+        // Check if already exists
+        const existing = this.findWorkspaceByPath(resolvedPath);
+        if (existing) {
+            return existing;
+        }
+        // Validate path exists
+        if (!fs.existsSync(resolvedPath)) {
+            throw new Error(`Path does not exist: ${resolvedPath}`);
+        }
+        const workspace = {
+            id: generateId(),
+            name: request.name || path.basename(resolvedPath),
+            path: resolvedPath,
+            status: 'inactive',
+            provider: request.provider || this.detectProvider(resolvedPath),
+            createdAt: new Date(),
+            lastActiveAt: new Date(),
+            ...this.getGitInfo(resolvedPath),
+        };
+        this.workspaces.set(workspace.id, workspace);
+        this.saveWorkspaces();
+        logger.info('Workspace added', { id: workspace.id, name: workspace.name });
+        this.broadcastEvent({
+            type: 'workspace:added',
+            workspaceId: workspace.id,
+            data: this.toPublicWorkspace(workspace),
+            timestamp: new Date(),
+        });
+        // Auto-start daemon
+        if (this.config.autoStartDaemons) {
+            this.startWorkspaceDaemon(workspace.id).catch((err) => {
+                logger.error('Failed to start workspace daemon', { id: workspace.id, error: String(err) });
+            });
+        }
+        return this.toPublicWorkspace(workspace);
+    }
+    /**
+     * Remove a workspace
+     */
+    async removeWorkspace(workspaceId) {
+        const workspace = this.workspaces.get(workspaceId);
+        if (!workspace)
+            return false;
+        // Stop daemon if running
+        await this.stopWorkspaceDaemon(workspaceId);
+        // Clear active if this was active
+        if (this.activeWorkspaceId === workspaceId) {
+            this.activeWorkspaceId = undefined;
+        }
+        this.workspaces.delete(workspaceId);
+        this.saveWorkspaces();
+        logger.info('Workspace removed', { id: workspaceId });
+        this.broadcastEvent({
+            type: 'workspace:removed',
+            workspaceId,
+            data: { id: workspaceId },
+            timestamp: new Date(),
+        });
+        return true;
+    }
+    /**
+     * Switch to a workspace
+     */
+    async switchWorkspace(workspaceId) {
+        const workspace = this.workspaces.get(workspaceId);
+        if (!workspace) {
+            throw new Error(`Workspace not found: ${workspaceId}`);
+        }
+        const previousId = this.activeWorkspaceId;
+        // Update status
+        if (previousId && previousId !== workspaceId) {
+            const prev = this.workspaces.get(previousId);
+            if (prev) {
+                prev.status = 'inactive';
+            }
+        }
+        workspace.status = 'active';
+        workspace.lastActiveAt = new Date();
+        this.activeWorkspaceId = workspaceId;
+        // Ensure daemon is running
+        if (!workspace.daemon?.isRunning) {
+            await this.startWorkspaceDaemon(workspaceId);
+        }
+        this.saveWorkspaces();
+        logger.info('Switched workspace', { id: workspaceId, name: workspace.name });
+        this.broadcastEvent({
+            type: 'workspace:switched',
+            workspaceId,
+            data: { previousId, currentId: workspaceId },
+            timestamp: new Date(),
+        });
+        return this.toPublicWorkspace(workspace);
+    }
+    /**
+     * Get all workspaces
+     */
+    getWorkspaces() {
+        return Array.from(this.workspaces.values()).map((w) => this.toPublicWorkspace(w));
+    }
+    /**
+     * Get workspace by ID
+     */
+    getWorkspace(workspaceId) {
+        const workspace = this.workspaces.get(workspaceId);
+        return workspace ? this.toPublicWorkspace(workspace) : undefined;
+    }
+    /**
+     * Get active workspace
+     */
+    getActiveWorkspace() {
+        if (!this.activeWorkspaceId)
+            return undefined;
+        return this.getWorkspace(this.activeWorkspaceId);
+    }
+    // === Agent Management ===
+    /**
+     * Spawn an agent in a workspace
+     */
+    async spawnAgent(workspaceId, request) {
+        const workspace = this.workspaces.get(workspaceId);
+        if (!workspace) {
+            throw new Error(`Workspace not found: ${workspaceId}`);
+        }
+        // Ensure daemon is running
+        if (!workspace.daemon?.isRunning) {
+            await this.startWorkspaceDaemon(workspaceId);
+        }
+        // Ensure spawner exists
+        if (!workspace.spawner) {
+            workspace.spawner = new AgentSpawner({
+                projectRoot: workspace.path,
+                onMarkSpawning: (name) => workspace.daemon?.markSpawning(name),
+                onClearSpawning: (name) => workspace.daemon?.clearSpawning(name),
+            });
+        }
+        const result = await workspace.spawner.spawn({
+            name: request.name,
+            cli: this.getCliForProvider(request.provider || workspace.provider),
+            task: request.task || '',
+        });
+        if (!result.success) {
+            throw new Error(result.error || 'Failed to spawn agent');
+        }
+        const agent = {
+            id: generateId(),
+            name: request.name,
+            workspaceId,
+            provider: request.provider || workspace.provider,
+            status: 'running',
+            pid: result.pid,
+            task: request.task,
+            spawnedAt: new Date(),
+            restartCount: 0,
+        };
+        // Register for health monitoring if we have a PID
+        if (result.pid) {
+            this.registerAgentHealth(workspaceId, request.name, result.pid);
+        }
+        else {
+            logger.warn('Agent spawned without PID - health monitoring disabled', {
+                workspaceId,
+                agentName: request.name,
+            });
+        }
+        logger.info('Agent spawned', { id: agent.id, name: agent.name, workspaceId, pid: result.pid });
+        this.broadcastEvent({
+            type: 'agent:spawned',
+            workspaceId,
+            agentId: agent.id,
+            data: agent,
+            timestamp: new Date(),
+        });
+        return agent;
+    }
+    /**
+     * Stop an agent
+     */
+    async stopAgent(workspaceId, agentName) {
+        const workspace = this.workspaces.get(workspaceId);
+        if (!workspace?.spawner)
+            return false;
+        // Mark as releasing BEFORE stopping to prevent crash announcement
+        this.markAgentReleasing(workspaceId, agentName);
+        try {
+            const released = await workspace.spawner.release(agentName);
+            if (released) {
+                // Unregister from health monitoring after successful release
+                this.unregisterAgentHealth(workspaceId, agentName);
+                this.broadcastEvent({
+                    type: 'agent:stopped',
+                    workspaceId,
+                    data: { name: agentName },
+                    timestamp: new Date(),
+                });
+                logger.info('Agent stopped gracefully', { workspaceId, agentName });
+            }
+            else {
+                // Release failed - clear the releasing flag
+                const health = this.getAgentHealth(workspaceId, agentName);
+                if (health) {
+                    health.releasing = false;
+                }
+            }
+            return released;
+        }
+        catch (err) {
+            // Release threw an exception - clean up health tracking to avoid stuck state
+            this.unregisterAgentHealth(workspaceId, agentName);
+            logger.error('Agent release failed with exception', {
+                workspaceId,
+                agentName,
+                error: String(err),
+            });
+            throw err;
+        }
+    }
+    /**
+     * Get agents in a workspace
+     */
+    getAgents(workspaceId) {
+        const workspace = this.workspaces.get(workspaceId);
+        if (!workspace?.spawner)
+            return [];
+        return workspace.spawner.getActiveWorkers().map((w) => {
+            // Get health data for this agent
+            const health = this.getAgentHealth(workspaceId, w.name);
+            return {
+                id: w.name,
+                name: w.name,
+                workspaceId,
+                provider: this.detectProviderFromCli(w.cli),
+                status: 'running',
+                pid: w.pid,
+                task: w.task,
+                spawnedAt: new Date(w.spawnedAt),
+                lastHealthCheck: health?.lastHeartbeatAt,
+                rssBytes: health?.lastRssBytes,
+                cpuPercent: health?.lastCpuPercent,
+                restartCount: 0,
+            };
+        });
+    }
+    // === Private Methods ===
+    /**
+     * Start daemon for a workspace
+     */
+    async startWorkspaceDaemon(workspaceId) {
+        const workspace = this.workspaces.get(workspaceId);
+        if (!workspace)
+            return;
+        if (workspace.daemon?.isRunning)
+            return;
+        try {
+            const paths = getProjectPaths(workspace.path);
+            workspace.daemon = new Daemon({
+                socketPath: paths.socketPath,
+                teamDir: paths.teamDir,
+            });
+            await workspace.daemon.start();
+            workspace.status = 'active';
+            // Create spawner
+            workspace.spawner = new AgentSpawner({
+                projectRoot: workspace.path,
+                onMarkSpawning: (name) => workspace.daemon?.markSpawning(name),
+                onClearSpawning: (name) => workspace.daemon?.clearSpawning(name),
+            });
+            // Set up agent death notifications
+            workspace.spawner.setOnAgentDeath((info) => {
+                // Broadcast to dashboard via WebSocket
+                this.broadcastEvent({
+                    type: 'agent:crashed',
+                    workspaceId,
+                    data: {
+                        name: info.name,
+                        exitCode: info.exitCode,
+                        continuityAgentId: info.agentId,
+                        resumeInstructions: info.resumeInstructions,
+                    },
+                    timestamp: new Date(),
+                });
+                // Broadcast to all connected agents via relay
+                const message = info.agentId
+                    ? `AGENT DIED: "${info.name}" has crashed (exit code: ${info.exitCode}). Agent ID: ${info.agentId}. ${info.resumeInstructions}`
+                    : `AGENT DIED: "${info.name}" has crashed (exit code: ${info.exitCode}).`;
+                workspace.daemon?.broadcastSystemMessage(message, {
+                    agentName: info.name,
+                    exitCode: info.exitCode,
+                    agentId: info.agentId,
+                    resumeInstructions: info.resumeInstructions,
+                });
+                logger.warn('Agent died', {
+                    name: info.name,
+                    exitCode: info.exitCode,
+                    agentId: info.agentId,
+                });
+            });
+            logger.info('Workspace daemon started', { id: workspaceId, socket: paths.socketPath });
+        }
+        catch (err) {
+            workspace.status = 'error';
+            logger.error('Failed to start workspace daemon', { id: workspaceId, error: String(err) });
+            throw err;
+        }
+    }
+    /**
+     * Stop daemon for a workspace
+     */
+    async stopWorkspaceDaemon(workspaceId) {
+        const workspace = this.workspaces.get(workspaceId);
+        if (!workspace)
+            return;
+        // Mark all agents as releasing to prevent crash announcements
+        const workspaceHealth = this.getWorkspaceAgentHealth(workspaceId);
+        for (const health of workspaceHealth) {
+            this.markAgentReleasing(workspaceId, health.agentName);
+        }
+        // Release all agents first
+        if (workspace.spawner) {
+            await workspace.spawner.releaseAll();
+        }
+        // Clean up health monitoring for all agents in this workspace
+        for (const health of workspaceHealth) {
+            this.unregisterAgentHealth(workspaceId, health.agentName);
+        }
+        // Stop daemon
+        if (workspace.daemon) {
+            await workspace.daemon.stop();
+            workspace.daemon = undefined;
+        }
+        workspace.spawner = undefined;
+        workspace.status = 'inactive';
+        logger.info('Workspace daemon stopped', { id: workspaceId });
+    }
+    /**
+     * Handle HTTP request
+     */
+    async handleRequest(req, res) {
+        // CORS
+        res.setHeader('Access-Control-Allow-Origin', '*');
+        res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
+        res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+        if (req.method === 'OPTIONS') {
+            res.writeHead(204);
+            res.end();
+            return;
+        }
+        const url = new URL(req.url || '/', `http://${req.headers.host}`);
+        const pathname = url.pathname;
+        const method = req.method || 'GET';
+        try {
+            let response;
+            // Health check
+            if (pathname === '/' && method === 'GET') {
+                response = { status: 200, body: { status: 'ok', version: '1.0.0' } };
+            }
+            // Metrics
+            else if (pathname === '/metrics' && method === 'GET') {
+                res.setHeader('Content-Type', 'text/plain');
+                res.writeHead(200);
+                res.end(metrics.toPrometheus());
+                return;
+            }
+            // List workspaces
+            else if (pathname === '/workspaces' && method === 'GET') {
+                response = {
+                    status: 200,
+                    body: {
+                        workspaces: this.getWorkspaces(),
+                        activeWorkspaceId: this.activeWorkspaceId,
+                    },
+                };
+            }
+            // Add workspace
+            else if (pathname === '/workspaces' && method === 'POST') {
+                const body = await this.parseBody(req);
+                const workspace = this.addWorkspace(body);
+                response = { status: 201, body: workspace };
+            }
+            // Get workspace
+            else if (pathname.match(/^\/workspaces\/[^/]+$/) && method === 'GET') {
+                const id = pathname.split('/')[2];
+                const workspace = this.getWorkspace(id);
+                response = workspace
+                    ? { status: 200, body: workspace }
+                    : { status: 404, body: { error: 'Not found' } };
+            }
+            // Delete workspace
+            else if (pathname.match(/^\/workspaces\/[^/]+$/) && method === 'DELETE') {
+                const id = pathname.split('/')[2];
+                const removed = await this.removeWorkspace(id);
+                response = removed
+                    ? { status: 204, body: null }
+                    : { status: 404, body: { error: 'Not found' } };
+            }
+            // Switch workspace
+            else if (pathname.match(/^\/workspaces\/[^/]+\/switch$/) && method === 'POST') {
+                const id = pathname.split('/')[2];
+                const workspace = await this.switchWorkspace(id);
+                response = { status: 200, body: workspace };
+            }
+            // List agents in workspace
+            else if (pathname.match(/^\/workspaces\/[^/]+\/agents$/) && method === 'GET') {
+                const id = pathname.split('/')[2];
+                const agents = this.getAgents(id);
+                response = { status: 200, body: { agents, workspaceId: id } };
+            }
+            // Spawn agent
+            else if (pathname.match(/^\/workspaces\/[^/]+\/agents$/) && method === 'POST') {
+                const id = pathname.split('/')[2];
+                const body = await this.parseBody(req);
+                const agent = await this.spawnAgent(id, body);
+                response = { status: 201, body: agent };
+            }
+            // Stop agent
+            else if (pathname.match(/^\/workspaces\/[^/]+\/agents\/[^/]+$/) && method === 'DELETE') {
+                const parts = pathname.split('/');
+                const workspaceId = parts[2];
+                const agentName = parts[4];
+                const stopped = await this.stopAgent(workspaceId, agentName);
+                response = stopped
+                    ? { status: 204, body: null }
+                    : { status: 404, body: { error: 'Not found' } };
+            }
+            // Not found
+            else {
+                response = { status: 404, body: { error: 'Not found' } };
+            }
+            res.setHeader('Content-Type', 'application/json');
+            res.writeHead(response.status);
+            res.end(response.body ? JSON.stringify(response.body) : '');
+        }
+        catch (err) {
+            logger.error('Request error', { error: String(err) });
+            res.setHeader('Content-Type', 'application/json');
+            res.writeHead(500);
+            res.end(JSON.stringify({ error: String(err) }));
+        }
+    }
+    /**
+     * Handle WebSocket connection
+     */
+    handleWebSocket(ws, _req) {
+        logger.info('WebSocket client connected');
+        // Mark client as alive for ping/pong keepalive
+        this.clientAlive.set(ws, true);
+        // Handle pong responses
+        ws.on('pong', () => {
+            this.clientAlive.set(ws, true);
+        });
+        const session = {
+            userId: 'anonymous',
+            githubUsername: 'anonymous',
+            connectedAt: new Date(),
+            activeWorkspaceId: this.activeWorkspaceId,
+        };
+        this.sessions.set(ws, session);
+        // Send initial state
+        this.sendToClient(ws, {
+            type: 'init',
+            data: {
+                workspaces: this.getWorkspaces(),
+                activeWorkspaceId: this.activeWorkspaceId,
+                agents: this.activeWorkspaceId ? this.getAgents(this.activeWorkspaceId) : [],
+            },
+        });
+        ws.on('message', (data) => {
+            try {
+                const msg = JSON.parse(data.toString());
+                this.handleWebSocketMessage(ws, session, msg);
+            }
+            catch (err) {
+                logger.error('WebSocket message error', { error: String(err) });
+            }
+        });
+        ws.on('close', () => {
+            this.sessions.delete(ws);
+            logger.info('WebSocket client disconnected');
+        });
+    }
+    /**
+     * Handle WebSocket message
+     */
+    handleWebSocketMessage(ws, session, msg) {
+        switch (msg.type) {
+            case 'switch_workspace':
+                if (typeof msg.data === 'string') {
+                    this.switchWorkspace(msg.data)
+                        .then((workspace) => {
+                        session.activeWorkspaceId = workspace.id;
+                    })
+                        .catch((err) => {
+                        this.sendToClient(ws, { type: 'error', data: String(err) });
+                    });
+                }
+                break;
+            case 'ping':
+                this.sendToClient(ws, { type: 'pong' });
+                break;
+        }
+    }
+    /**
+     * Send to WebSocket client
+     */
+    sendToClient(ws, msg) {
+        if (ws.readyState === WebSocket.OPEN) {
+            ws.send(JSON.stringify(msg));
+        }
+    }
+    /**
+     * Broadcast event to all clients
+     */
+    broadcastEvent(event) {
+        if (!this.wss)
+            return;
+        const msg = JSON.stringify({ type: 'event', data: event });
+        for (const ws of this.wss.clients) {
+            if (ws.readyState === WebSocket.OPEN) {
+                ws.send(msg);
+            }
+        }
+    }
+    /**
+     * Parse request body
+     */
+    parseBody(req) {
+        return new Promise((resolve, reject) => {
+            let data = '';
+            req.on('data', (chunk) => (data += chunk));
+            req.on('end', () => {
+                try {
+                    resolve(data ? JSON.parse(data) : {});
+                }
+                catch {
+                    reject(new Error('Invalid JSON'));
+                }
+            });
+        });
+    }
+    /**
+     * Load workspaces from disk
+     */
+    loadWorkspaces() {
+        if (!fs.existsSync(this.workspacesFile))
+            return;
+        try {
+            const data = JSON.parse(fs.readFileSync(this.workspacesFile, 'utf8'));
+            for (const w of data.workspaces || []) {
+                this.workspaces.set(w.id, {
+                    ...w,
+                    createdAt: new Date(w.createdAt),
+                    lastActiveAt: new Date(w.lastActiveAt),
+                    status: 'inactive',
+                });
+            }
+            this.activeWorkspaceId = data.activeWorkspaceId;
+            logger.info('Loaded workspaces', { count: this.workspaces.size });
+        }
+        catch (err) {
+            logger.error('Failed to load workspaces', { error: String(err) });
+        }
+    }
+    /**
+     * Save workspaces to disk
+     */
+    saveWorkspaces() {
+        try {
+            const data = {
+                workspaces: Array.from(this.workspaces.values()).map((w) => this.toPublicWorkspace(w)),
+                activeWorkspaceId: this.activeWorkspaceId,
+            };
+            fs.writeFileSync(this.workspacesFile, JSON.stringify(data, null, 2));
+        }
+        catch (err) {
+            logger.error('Failed to save workspaces', { error: String(err) });
+        }
+    }
+    /**
+     * Find workspace by path
+     */
+    findWorkspaceByPath(path) {
+        const resolved = this.resolvePath(path);
+        const workspace = Array.from(this.workspaces.values()).find((w) => w.path === resolved);
+        return workspace ? this.toPublicWorkspace(workspace) : undefined;
+    }
+    /**
+     * Resolve path
+     */
+    resolvePath(p) {
+        if (p.startsWith('~')) {
+            p = path.join(process.env.HOME || '', p.slice(1));
+        }
+        return path.resolve(p);
+    }
+    /**
+     * Detect provider from workspace
+     */
+    detectProvider(workspacePath) {
+        if (fs.existsSync(path.join(workspacePath, 'CLAUDE.md')) ||
+            fs.existsSync(path.join(workspacePath, '.claude'))) {
+            return 'claude';
+        }
+        if (fs.existsSync(path.join(workspacePath, '.codex'))) {
+            return 'codex';
+        }
+        if (fs.existsSync(path.join(workspacePath, '.gemini'))) {
+            return 'gemini';
+        }
+        return 'generic';
+    }
+    /**
+     * Detect provider from CLI command
+     */
+    detectProviderFromCli(cli) {
+        if (cli.includes('claude'))
+            return 'claude';
+        if (cli.includes('codex'))
+            return 'codex';
+        if (cli.includes('gemini'))
+            return 'gemini';
+        return 'generic';
+    }
+    /**
+     * Get CLI command for provider
+     */
+    getCliForProvider(provider) {
+        switch (provider) {
+            case 'claude':
+                return 'claude';
+            case 'codex':
+                return 'codex';
+            case 'gemini':
+                return 'gemini';
+            default:
+                return 'claude';
+        }
+    }
+    /**
+     * Get git info
+     */
+    getGitInfo(workspacePath) {
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-require-imports
+            const { execSync } = require('child_process');
+            const branch = execSync('git branch --show-current', {
+                cwd: workspacePath,
+                encoding: 'utf8',
+                stdio: ['pipe', 'pipe', 'pipe'],
+            }).trim();
+            let remote;
+            try {
+                remote = execSync('git remote get-url origin', {
+                    cwd: workspacePath,
+                    encoding: 'utf8',
+                    stdio: ['pipe', 'pipe', 'pipe'],
+                }).trim();
+            }
+            catch {
+                // No remote
+            }
+            return { gitRemote: remote, gitBranch: branch };
+        }
+        catch {
+            return {};
+        }
+    }
+    /**
+     * Convert to public workspace (without internal references)
+     */
+    toPublicWorkspace(w) {
+        return {
+            id: w.id,
+            name: w.name,
+            path: w.path,
+            status: w.status,
+            provider: w.provider,
+            createdAt: w.createdAt,
+            lastActiveAt: w.lastActiveAt,
+            cloudId: w.cloudId,
+            customDomain: w.customDomain,
+            gitRemote: w.gitRemote,
+            gitBranch: w.gitBranch,
+        };
+    }
+    // === Health Monitoring ===
+    /**
+     * Start agent health monitoring.
+     * Monitors PIDs for liveness and tracks memory/CPU usage.
+     */
+    startHealthMonitoring() {
+        // Start the memory monitor
+        this.memoryMonitor.start();
+        // Listen for memory samples to update health state
+        // Store handler reference for cleanup
+        this.memorySampleHandler = (event) => {
+            const health = this.agentHealth.get(event.name);
+            if (health) {
+                health.lastSampleAt = new Date();
+                health.lastRssBytes = event.snapshot.rssBytes;
+                health.lastCpuPercent = event.snapshot.cpuPercent;
+                // Check for high CPU usage and broadcast alert
+                if (event.snapshot.cpuPercent >= CPU_ALERT_THRESHOLD) {
+                    this.broadcastResourceAlert(health, 'cpu', event.snapshot.cpuPercent);
+                }
+            }
+        };
+        this.memoryMonitor.on('sample', this.memorySampleHandler);
+        // Listen for memory alerts and broadcast to agents
+        // Store handler reference for cleanup
+        this.memoryAlertHandler = (alert) => {
+            const health = this.agentHealth.get(alert.agentName);
+            if (health && alert.type !== 'recovered') {
+                this.broadcastResourceAlert(health, 'memory', alert.currentRss, alert);
+            }
+        };
+        this.memoryMonitor.on('alert', this.memoryAlertHandler);
+        // Start heartbeat interval to check PIDs are alive
+        this.heartbeatInterval = setInterval(() => {
+            this.checkAgentHeartbeats();
+        }, HEARTBEAT_INTERVAL_MS);
+        logger.info('Health monitoring started', {
+            heartbeatIntervalMs: HEARTBEAT_INTERVAL_MS,
+            cpuAlertThreshold: CPU_ALERT_THRESHOLD,
+        });
+    }
+    /**
+     * Check all registered agents' PIDs are still alive.
+     * If a PID has died unexpectedly, broadcast a crash notification.
+     */
+    checkAgentHeartbeats() {
+        // Collect crashed agents first to avoid modifying map during iteration
+        const crashedAgents = [];
+        for (const [key, health] of this.agentHealth) {
+            const isAlive = this.isProcessAlive(health.pid);
+            if (isAlive) {
+                // Only update heartbeat timestamp for alive processes
+                health.lastHeartbeatAt = new Date();
+            }
+            else if (!health.releasing) {
+                // Agent died unexpectedly - mark for crash handling
+                // Immediately remove from map to prevent duplicate handling on next interval
+                this.agentHealth.delete(key);
+                crashedAgents.push(health);
+            }
+            // If !isAlive && health.releasing, agent is being gracefully stopped - skip
+        }
+        // Now handle crashes outside the iteration
+        for (const health of crashedAgents) {
+            logger.warn('Agent heartbeat failed - process died', {
+                workspaceId: health.workspaceId,
+                agentName: health.agentName,
+                pid: health.pid,
+            });
+            this.handleAgentCrash(health);
+        }
+    }
+    /**
+     * Check if a process is alive by sending signal 0.
+     */
+    isProcessAlive(pid) {
+        try {
+            process.kill(pid, 0);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Handle an agent crash - unregister and broadcast to other agents.
+     * Note: Agent is already removed from agentHealth map before this is called.
+     */
+    handleAgentCrash(health) {
+        const workspace = this.workspaces.get(health.workspaceId);
+        // Get crash context from memory monitor for analysis
+        const crashContext = this.memoryMonitor.getCrashContext(health.agentName);
+        // Unregister from memory monitor (agent already removed from agentHealth map)
+        this.memoryMonitor.unregister(health.agentName);
+        // Broadcast crash to dashboard via WebSocket
+        this.broadcastEvent({
+            type: 'agent:crashed',
+            workspaceId: health.workspaceId,
+            data: {
+                name: health.agentName,
+                pid: health.pid,
+                crashContext: {
+                    likelyCause: crashContext.likelyCause,
+                    peakMemory: crashContext.peakMemory,
+                    averageMemory: crashContext.averageMemory,
+                    memoryTrend: crashContext.memoryTrend,
+                    analysisNotes: crashContext.analysisNotes,
+                },
+            },
+            timestamp: new Date(),
+        });
+        // Broadcast to all connected agents in the workspace via relay
+        const message = crashContext.likelyCause !== 'unknown'
+            ? `AGENT CRASHED: "${health.agentName}" has died unexpectedly (PID: ${health.pid}). Likely cause: ${crashContext.likelyCause}. ${crashContext.analysisNotes.slice(0, 2).join('. ')}`
+            : `AGENT CRASHED: "${health.agentName}" has died unexpectedly (PID: ${health.pid}).`;
+        workspace?.daemon?.broadcastSystemMessage(message, {
+            agentName: health.agentName,
+            pid: health.pid,
+            likelyCause: crashContext.likelyCause,
+            crashType: 'heartbeat_failure',
+        });
+        logger.error('Agent crashed', {
+            workspaceId: health.workspaceId,
+            agentName: health.agentName,
+            pid: health.pid,
+            likelyCause: crashContext.likelyCause,
+        });
+    }
+    /**
+     * Broadcast a resource alert (memory or CPU) to agents.
+     */
+    broadcastResourceAlert(health, resourceType, currentValue, memoryAlert) {
+        // CPU alert cooldown to avoid spamming
+        if (resourceType === 'cpu') {
+            const now = Date.now();
+            if (health.lastCpuAlertAt && now - health.lastCpuAlertAt < RESOURCE_ALERT_COOLDOWN_MS) {
+                return; // Still in cooldown
+            }
+            health.lastCpuAlertAt = now;
+        }
+        const workspace = this.workspaces.get(health.workspaceId);
+        // Broadcast to dashboard
+        this.broadcastEvent({
+            type: 'agent:resource-alert',
+            workspaceId: health.workspaceId,
+            agentId: health.agentName,
+            data: {
+                name: health.agentName,
+                resourceType,
+                currentValue,
+                alertLevel: memoryAlert?.type ?? 'high_cpu',
+                message: memoryAlert?.message ??
+                    `Agent "${health.agentName}" is running at ${currentValue.toFixed(1)}% CPU`,
+                recommendation: memoryAlert?.recommendation ??
+                    'Consider reducing workload or checking for runaway processes',
+            },
+            timestamp: new Date(),
+        });
+        // Broadcast to agents
+        const message = resourceType === 'memory'
+            ? `RESOURCE ALERT: "${health.agentName}" memory usage is ${memoryAlert?.type ?? 'high'} (${formatBytes(currentValue)}). ${memoryAlert?.recommendation ?? ''}`
+            : `RESOURCE ALERT: "${health.agentName}" is running at ${currentValue.toFixed(1)}% CPU. Consider reducing workload.`;
+        workspace?.daemon?.broadcastSystemMessage(message, {
+            agentName: health.agentName,
+            resourceType,
+            alertLevel: memoryAlert?.type ?? 'high_cpu',
+        });
+        logger.warn('Resource alert', {
+            workspaceId: health.workspaceId,
+            agentName: health.agentName,
+            resourceType,
+            currentValue: resourceType === 'memory' ? formatBytes(currentValue) : `${currentValue.toFixed(1)}%`,
+            alertLevel: memoryAlert?.type ?? 'high_cpu',
+        });
+    }
+    /**
+     * Register an agent for health monitoring.
+     */
+    registerAgentHealth(workspaceId, agentName, pid) {
+        const key = `${workspaceId}:${agentName}`;
+        // Guard against double-registration - update PID instead
+        if (this.agentHealth.has(key)) {
+            logger.warn('Agent already registered for health monitoring, updating PID', {
+                workspaceId,
+                agentName,
+                newPid: pid,
+            });
+            this.updateAgentHealthPid(workspaceId, agentName, pid);
+            return;
+        }
+        this.agentHealth.set(key, {
+            key,
+            workspaceId,
+            agentName,
+            pid,
+            lastHeartbeatAt: new Date(),
+        });
+        // Register with memory monitor
+        this.memoryMonitor.register(agentName, pid);
+        logger.info('Agent registered for health monitoring', {
+            workspaceId,
+            agentName,
+            pid,
+        });
+    }
+    /**
+     * Update PID for an agent (after restart).
+     *
+     * This method is intended for agent restart scenarios where the agent process
+     * is restarted with a new PID but should maintain continuity in health tracking.
+     * Currently unused but reserved for future auto-restart functionality.
+     *
+     * @param workspaceId - The workspace ID
+     * @param agentName - The agent name
+     * @param newPid - The new process ID after restart
+     */
+    updateAgentHealthPid(workspaceId, agentName, newPid) {
+        const key = `${workspaceId}:${agentName}`;
+        const health = this.agentHealth.get(key);
+        if (health) {
+            health.pid = newPid;
+            health.releasing = false;
+            health.lastHeartbeatAt = new Date();
+            this.memoryMonitor.updatePid(agentName, newPid);
+            logger.info('Agent health PID updated', {
+                workspaceId,
+                agentName,
+                newPid,
+            });
+        }
+        else {
+            // Register new
+            this.registerAgentHealth(workspaceId, agentName, newPid);
+        }
+    }
+    /**
+     * Mark an agent as releasing (to avoid crash announcement).
+     */
+    markAgentReleasing(workspaceId, agentName) {
+        const key = `${workspaceId}:${agentName}`;
+        const health = this.agentHealth.get(key);
+        if (health) {
+            health.releasing = true;
+            logger.debug('Agent marked as releasing', { workspaceId, agentName });
+        }
+    }
+    /**
+     * Unregister an agent from health monitoring.
+     */
+    unregisterAgentHealth(workspaceId, agentName) {
+        const key = `${workspaceId}:${agentName}`;
+        this.agentHealth.delete(key);
+        this.memoryMonitor.unregister(agentName);
+        logger.debug('Agent unregistered from health monitoring', {
+            workspaceId,
+            agentName,
+        });
+    }
+    /**
+     * Get health state for an agent.
+     */
+    getAgentHealth(workspaceId, agentName) {
+        return this.agentHealth.get(`${workspaceId}:${agentName}`);
+    }
+    /**
+     * Get health states for all agents in a workspace.
+     */
+    getWorkspaceAgentHealth(workspaceId) {
+        return Array.from(this.agentHealth.values()).filter((h) => h.workspaceId === workspaceId);
+    }
+}
+let orchestratorInstance;
+/**
+ * Start the orchestrator
+ */
+export async function startOrchestrator(config = {}) {
+    if (orchestratorInstance) {
+        return orchestratorInstance;
+    }
+    orchestratorInstance = new Orchestrator(config);
+    await orchestratorInstance.start();
+    return orchestratorInstance;
+}
+/**
+ * Stop the orchestrator
+ */
+export async function stopOrchestrator() {
+    if (orchestratorInstance) {
+        await orchestratorInstance.stop();
+        orchestratorInstance = undefined;
+    }
+}
+/**
+ * Get orchestrator instance
+ */
+export function getOrchestrator() {
+    return orchestratorInstance;
+}
+//# sourceMappingURL=orchestrator.js.map