@mxf-dev/core 2.0.4 → 3.0.0

package/dist/protocols/mcp/services/ExternalMcpServerManager.js

@@ -34,6 +34,7 @@ import { createExternalMcpServerEventPayload, createExternalMcpServerErrorEventP
 import { AutoCorrectionService } from '../../../services/AutoCorrectionService.js';
 import { Events } from '../../../events/EventNames.js';
 import { v4 as uuidv4 } from 'uuid';
+import { assertUnsafeStdioMcpEnabled } from '../security/ExternalMcpRegistrationPolicy.js';
 // Create logger and validator instances
 const logger = new Logger('info', 'ExternalMcpServerManager', 'server');
 const validator = createStrictValidator('ExternalMcpServerManager');
@@ -44,6 +45,15 @@ const REQUEST_TIMEOUTS_MS = {
     'tools/call': 30000,
     ping: 5000
 };
+/** Delay before restarting a crashed server. */
+const DEFAULT_RESTART_DELAY_MS = 2000;
+/**
+ * Consecutive failed health probes after which a running server is declared
+ * dead and restarted. One failure can be a slow reply; two in a row on a
+ * connection that answers in milliseconds when healthy means the connection
+ * is gone.
+ */
+const HEALTH_FAILURES_BEFORE_RESTART = 2;
 /** MCP protocol version this client negotiates in the initialize handshake. */
 const MCP_PROTOCOL_VERSION = '2024-11-05';
 /** Version this client reports to servers. */
@@ -55,14 +65,28 @@ export class ExternalMcpServerManager extends EventEmitter {
     servers = new Map();
     // Scope tracking for channel/agent-scoped servers
     serverScopes = new Map();
+    /**
+     * Channel ids which crossed their terminal deletion boundary in this
+     * process. Channel ids are reserved in persistence, so this tombstone never
+     * needs to be removed and closes register/delete races in both directions.
+     */
+    retiredChannelIds = new Set();
     autoCorrectionService;
     /** Injectable event emitter — decouples from direct EventBus.server dependency.
      *  Server-side: uses default ServerToolEventEmitter (wraps EventBus.server).
      *  Client-side: uses ClientToolEventEmitter (wraps EventBus.client + socketEmit). */
     toolEventEmitter = null;
+    /** EventBus subscriptions owned by this manager instance. */
+    eventSubscriptions = [];
+    /** Delay before restarting a crashed server. Overridable for tests. */
+    restartDelayMs = DEFAULT_RESTART_DELAY_MS;
+    /** Per-method JSON-RPC reply timeouts. Overridable for tests and tuning. */
+    requestTimeouts = { ...REQUEST_TIMEOUTS_MS };
     /**
      * @param options.toolEventEmitter - Injectable event emitter for decoupling from EventBus.server
      * @param options.skipServerEventHandlers - Skip setting up EventBus.server listeners (for client-side usage)
+     * @param options.restartDelayMs - Delay before restarting a crashed server (default 2000)
+     * @param options.requestTimeoutsMs - Per-method JSON-RPC reply timeout overrides
      */
     constructor(options) {
         super();
@@ -70,6 +94,12 @@ export class ExternalMcpServerManager extends EventEmitter {
         if (options?.toolEventEmitter) {
             this.toolEventEmitter = options.toolEventEmitter;
         }
+        if (options?.restartDelayMs !== undefined) {
+            this.restartDelayMs = options.restartDelayMs;
+        }
+        if (options?.requestTimeoutsMs) {
+            this.requestTimeouts = { ...this.requestTimeouts, ...options.requestTimeoutsMs };
+        }
         // Set up event listeners for SDK-initiated server registration
         // Skipped when running client-side (no EventBus.server listeners needed)
         if (!options?.skipServerEventHandlers) {
@@ -81,8 +111,12 @@ export class ExternalMcpServerManager extends EventEmitter {
      */
     setupEventHandlers() {
         // Handle external server registration requests from SDK
-        EventBus.server.on(Events.Mcp.EXTERNAL_SERVER_REGISTER, async (payload) => {
+        this.eventSubscriptions.push(EventBus.server.on(Events.Mcp.EXTERNAL_SERVER_REGISTER, async (payload) => {
             try {
+                // EventBus registrations originate outside this manager (normally
+                // from a socket). They are caller-controlled and stdio means host
+                // process execution, so require the operator's explicit opt-in.
+                assertUnsafeStdioMcpEnabled(payload.data?.transport);
                 const serverConfig = {
                     id: payload.data.id,
                     name: payload.data.name,
@@ -130,13 +164,12 @@ export class ExternalMcpServerManager extends EventEmitter {
                     }
                 });
             }
-        });
+        }));
         // Handle external server unregistration requests from SDK
-        EventBus.server.on(Events.Mcp.EXTERNAL_SERVER_UNREGISTER, async (payload) => {
+        this.eventSubscriptions.push(EventBus.server.on(Events.Mcp.EXTERNAL_SERVER_UNREGISTER, async (payload) => {
             try {
                 const serverId = payload.data.serverId;
-                await this.stopServer(serverId);
-                this.servers.delete(serverId);
+                await this.unregisterServer(serverId);
                 // Emit success response
                 EventBus.server.emit(Events.Mcp.EXTERNAL_SERVER_UNREGISTERED, {
                     eventId: uuidv4(),
@@ -152,7 +185,9 @@ export class ExternalMcpServerManager extends EventEmitter {
                 });
             }
             catch (error) {
-                logger.error(`Error unregistering external server:`, error);
+                // Interpolate the message: passing the raw error as a second console
+                // argument makes Bun print a source-code excerpt instead of the message.
+                logger.error(`Error unregistering external server: ${error instanceof Error ? error.message : String(error)}`);
                 // Emit error response
                 EventBus.server.emit(Events.Mcp.EXTERNAL_SERVER_UNREGISTERED, {
                     eventId: uuidv4(),
@@ -167,50 +202,40 @@ export class ExternalMcpServerManager extends EventEmitter {
                     }
                 });
             }
-        });
+        }));
         // Handle channel-scoped server registration requests
-        EventBus.server.on(Events.Mcp.CHANNEL_SERVER_REGISTER, async (payload) => {
+        this.eventSubscriptions.push(EventBus.server.on(Events.Mcp.CHANNEL_SERVER_REGISTER, async (payload) => {
             try {
+                // Channel membership is not permission to execute a host command.
+                // The HTTP/socket entry point must also establish administrator
+                // authority; this is the defense-in-depth feature gate.
+                assertUnsafeStdioMcpEnabled(payload.data?.transport);
                 logger.info(`[CHANNEL_SERVER_REGISTER] Received registration request: ${JSON.stringify({ agentId: payload.agentId, channelId: payload.channelId, serverId: payload.data?.id })}`);
                 const channelId = payload.data.channelId || payload.channelId;
-                const serverId = `${channelId}:${payload.data.id}`;
-                const serverConfig = {
-                    id: serverId,
+                await this.registerChannelServer(channelId, {
+                    id: payload.data.id,
                     name: payload.data.name,
                     version: payload.data.version || '1.0.0',
                     command: payload.data.command || '',
                     args: payload.data.args || [],
-                    transport: (payload.data.transport || 'stdio'),
-                    url: payload.data.url,
                     autoStart: payload.data.autoStart !== false,
                     restartOnCrash: payload.data.restartOnCrash !== false,
                     maxRestartAttempts: payload.data.maxRestartAttempts || 3,
                     healthCheckInterval: payload.data.healthCheckInterval || 30000,
                     startupTimeout: payload.data.startupTimeout || 10000,
-                    environmentVariables: payload.data.environmentVariables || {}
-                };
-                // Track scope BEFORE registration so we can store the registration context
-                // Note: connectedAgents starts empty - only actual game agents are counted, not the admin who registers
-                this.serverScopes.set(serverId, {
-                    scope: 'channel',
-                    scopeId: channelId,
-                    connectedAgents: new Set(),
-                    keepAliveMinutes: payload.data.keepAliveMinutes || 5,
-                    // Store registration context for deferred success emission
-                    registrationContext: {
-                        agentId: payload.agentId,
-                        channelId,
-                        originalServerId: payload.data.id,
-                        serverName: payload.data.name
-                    }
+                    environmentVariables: payload.data.environmentVariables || {},
+                    keepAliveMinutes: payload.data.keepAliveMinutes
+                }, 
+                // Registration context for deferred success emission after tool discovery
+                {
+                    agentId: payload.agentId,
+                    channelId,
+                    originalServerId: payload.data.id,
+                    serverName: payload.data.name
                 });
-                // Register the server (this starts the process and tool discovery)
-                // Success event will be emitted after tool discovery completes
-                await this.registerServer(serverConfig);
-                logger.info(`[CHANNEL_SERVER_REGISTER] Server ${serverId} registered, waiting for tool discovery before emitting success`);
             }
             catch (error) {
-                logger.error(`Error registering channel server:`, error);
+                logger.error(`Error registering channel server: ${error instanceof Error ? error.message : String(error)}`);
                 // Emit error response
                 EventBus.server.emit(McpEvents.CHANNEL_SERVER_REGISTRATION_FAILED, createExternalMcpServerEventPayload(McpEvents.CHANNEL_SERVER_REGISTRATION_FAILED, payload.agentId, payload.data?.channelId || payload.channelId, {
                     serverId: payload.data?.id,
@@ -220,15 +245,12 @@ export class ExternalMcpServerManager extends EventEmitter {
                     error: error instanceof Error ? error.message : String(error)
                 }));
             }
-        });
+        }));
         // Handle channel server unregistration requests
-        EventBus.server.on(Events.Mcp.CHANNEL_SERVER_UNREGISTER, async (payload) => {
+        this.eventSubscriptions.push(EventBus.server.on(Events.Mcp.CHANNEL_SERVER_UNREGISTER, async (payload) => {
             try {
                 const channelId = payload.data.channelId || payload.channelId;
-                const serverId = `${channelId}:${payload.data.serverId}`;
-                await this.stopServer(serverId);
-                this.servers.delete(serverId);
-                this.serverScopes.delete(serverId);
+                await this.unregisterChannelServer(channelId, payload.data.serverId);
                 // Emit success response
                 EventBus.server.emit(McpEvents.CHANNEL_SERVER_UNREGISTERED, createExternalMcpServerEventPayload(McpEvents.CHANNEL_SERVER_UNREGISTERED, payload.agentId, channelId, {
                     serverId: payload.data.serverId,
@@ -239,7 +261,7 @@ export class ExternalMcpServerManager extends EventEmitter {
                 }));
             }
             catch (error) {
-                logger.error(`Error unregistering channel server:`, error);
+                logger.error(`Error unregistering channel server: ${error instanceof Error ? error.message : String(error)}`);
                 // Emit error response
                 EventBus.server.emit(McpEvents.CHANNEL_SERVER_UNREGISTERED, createExternalMcpServerEventPayload(McpEvents.CHANNEL_SERVER_UNREGISTERED, payload.agentId, payload.data?.channelId || payload.channelId, {
                     serverId: payload.data?.serverId,
@@ -249,12 +271,18 @@ export class ExternalMcpServerManager extends EventEmitter {
                     error: error instanceof Error ? error.message : String(error)
                 }));
             }
-        });
+        }));
     }
     /**
      * Register a new external server configuration
      */
     async registerServer(config) {
+        // This manager currently speaks line-delimited MCP over child-process
+        // stdio only. Never let an HTTP-labelled registration fall through to
+        // spawn(config.command, ...), which would bypass the stdio feature gate.
+        if (config.transport === 'http') {
+            throw new Error('HTTP transport registration is not implemented by ExternalMcpServerManager');
+        }
         logger.info(`[REGISTER_SERVER] Registering server ${config.id}: command="${config.command}", args=${JSON.stringify(config.args)}, autoStart=${config.autoStart}`);
         // Validate configuration
         validator.assertIsNonEmptyString(config.id, 'Server ID must be a non-empty string');
@@ -287,7 +315,203 @@ export class ExternalMcpServerManager extends EventEmitter {
         }
     }
     /**
-     * Start an external server process
+     * Register a channel-scoped server: track its scope, then register and
+     * (per config) start it. Resolves after the MCP handshake and tool
+     * discovery when autoStart is set, so a resolved promise means the tools
+     * are in the registry.
+     *
+     * A re-registration over an existing scope preserves the connected-agent
+     * set and clears any pending keepAlive timer — previously the timer
+     * reference was silently overwritten while the timer kept running, so a
+     * stale keepAlive could stop a freshly re-registered server later.
+     */
+    async registerChannelServer(channelId, config, registrationContext) {
+        validator.assertIsNonEmptyString(channelId, 'channelId must be a non-empty string');
+        const serverId = `${channelId}:${config.id}`;
+        const keepAliveMinutes = config.keepAliveMinutes || 5;
+        if (this.retiredChannelIds.has(channelId)) {
+            throw new Error(`Channel ${channelId} is deleted; MCP servers cannot be registered`);
+        }
+        const existingScope = this.serverScopes.get(serverId);
+        if (existingScope?.keepAliveTimer) {
+            clearTimeout(existingScope.keepAliveTimer);
+        }
+        this.serverScopes.set(serverId, {
+            scope: 'channel',
+            scopeId: channelId,
+            // Only actual channel agents are counted, not the admin who registers.
+            // On re-registration, keep whoever is already connected.
+            connectedAgents: existingScope?.connectedAgents ?? new Set(),
+            keepAliveMinutes,
+            registrationContext
+        });
+        logger.info(`[CHANNEL_SERVER_REGISTER] Registering channel server ${serverId} ` +
+            `(keepAlive ${keepAliveMinutes}min, autoStart ${config.autoStart}, restartOnCrash ${config.restartOnCrash})`);
+        const { keepAliveMinutes: _ignored, ...serverConfig } = config;
+        try {
+            await this.registerServer({ ...serverConfig, id: serverId });
+            // Deletion may have won while process startup/handshake was in
+            // flight. Remove the just-created runtime before returning so a
+            // late registration cannot resurrect a deleted channel's server.
+            if (this.retiredChannelIds.has(channelId)) {
+                await this.removeServer(serverId, 'registration raced channel deletion');
+                throw new Error(`Channel ${channelId} was deleted during MCP server registration`);
+            }
+        }
+        catch (error) {
+            // Registration failed before the server record existed — do not leave
+            // a scope entry behind for agents to "join" (unless one existed before).
+            if (!existingScope && !this.servers.has(serverId)) {
+                this.serverScopes.delete(serverId);
+            }
+            throw error;
+        }
+    }
+    /**
+     * Unregister a channel-scoped server: stop the process and remove both the
+     * server record and the scope tracking, including any keepAlive timer.
+     *
+     * Idempotent: unregistering a server that is partially or fully gone cleans
+     * up whatever remains and resolves. Production showed the half-removed
+     * state — record deleted, scope alive — and an unregister that throws
+     * "not found" against it leaves the zombie in place forever.
+     */
+    async unregisterChannelServer(channelId, serverId) {
+        validator.assertIsNonEmptyString(channelId, 'channelId must be a non-empty string');
+        validator.assertIsNonEmptyString(serverId, 'serverId must be a non-empty string');
+        await this.removeServer(`${channelId}:${serverId}`, 'channel server unregistration');
+    }
+    /**
+     * Permanently retire a channel's MCP runtime in this process.
+     *
+     * The tombstone is installed before process cleanup, which makes concurrent
+     * registrations fail both before and after their asynchronous startup.
+     * Every known scoped record is removed, including orphaned records whose
+     * scope metadata or server entry is only partially present.
+     */
+    async retireChannel(channelId) {
+        validator.assertIsNonEmptyString(channelId, 'channelId must be a non-empty string');
+        this.retiredChannelIds.add(channelId);
+        const serverIds = new Set();
+        for (const [serverId, scopeData] of this.serverScopes.entries()) {
+            if (scopeData.scope === 'channel' && scopeData.scopeId === channelId) {
+                serverIds.add(serverId);
+            }
+        }
+        for (const serverId of this.servers.keys()) {
+            if (serverId.startsWith(`${channelId}:`)) {
+                serverIds.add(serverId);
+            }
+        }
+        const results = await Promise.allSettled(Array.from(serverIds, serverId => (this.removeServer(serverId, 'channel deletion'))));
+        const failures = results.filter((result) => result.status === 'rejected');
+        if (failures.length > 0) {
+            throw new Error(`Failed to retire ${failures.length} MCP server(s) for channel ${channelId}`);
+        }
+    }
+    /**
+     * Unregister a server by its full id (global servers, and the SDK-facing
+     * EXTERNAL_SERVER_UNREGISTER path). Also removes scope tracking — this
+     * path used to delete only the server record, which was exactly the
+     * zombie state observed in production: agents kept "joining" a scope
+     * whose server no longer existed.
+     */
+    async unregisterServer(serverId) {
+        validator.assertIsNonEmptyString(serverId, 'serverId must be a non-empty string');
+        await this.removeServer(serverId, 'server unregistration');
+    }
+    /**
+     * Stop a server (if running) and remove every trace of it: server record,
+     * scope entry, keepAlive timer. Never throws for a missing record — it
+     * removes what exists and says what it did.
+     */
+    async removeServer(serverId, reason) {
+        const serverData = this.servers.get(serverId);
+        const scopeData = this.serverScopes.get(serverId);
+        if (!serverData && !scopeData) {
+            logger.warn(`Nothing to unregister for ${serverId} (${reason}) — no record, no scope`);
+            return;
+        }
+        if (scopeData?.keepAliveTimer) {
+            clearTimeout(scopeData.keepAliveTimer);
+            scopeData.keepAliveTimer = undefined;
+        }
+        if (serverData) {
+            try {
+                await this.stopServer(serverId, undefined, undefined, reason);
+            }
+            catch (error) {
+                logger.error(`Error stopping ${serverId} during ${reason}: ` +
+                    `${error instanceof Error ? error.message : String(error)} — removing its record anyway`);
+            }
+        }
+        else {
+            logger.warn(`Unregistering ${serverId} (${reason}): server record was already gone, ` +
+                `removing the orphaned scope entry`);
+        }
+        this.servers.delete(serverId);
+        this.serverScopes.delete(serverId);
+        logger.info(`Unregistered server ${serverId} (${reason})`);
+    }
+    /**
+     * Remove a server that cannot be kept alive (restart budget exhausted,
+     * unrecoverable spawn failure). Loud by design: this is the path that
+     * prevents zombies, so it reports at error level.
+     */
+    removeServerAfterFailure(serverId, reason) {
+        const serverData = this.servers.get(serverId);
+        if (serverData) {
+            if (serverData.healthCheckTimer) {
+                clearInterval(serverData.healthCheckTimer);
+                serverData.healthCheckTimer = undefined;
+            }
+            if (serverData.startupTimer) {
+                clearTimeout(serverData.startupTimer);
+                serverData.startupTimer = undefined;
+            }
+            this.rejectPendingRequests(serverId, reason);
+            this.settleReadyWaiters(serverId, new Error(reason));
+            if (serverData.process && !serverData.process.killed) {
+                serverData.process.kill('SIGKILL');
+            }
+        }
+        const scopeData = this.serverScopes.get(serverId);
+        if (scopeData?.keepAliveTimer) {
+            clearTimeout(scopeData.keepAliveTimer);
+        }
+        this.servers.delete(serverId);
+        this.serverScopes.delete(serverId);
+        logger.error(`Server ${serverId} unregistered: ${reason}. ` +
+            `Its tools are removed from the registry; re-register the server to restore them.`);
+        this.emitServerEvent(McpEvents.EXTERNAL_SERVER_STOPPED, serverId);
+    }
+    /**
+     * Settle every caller waiting for a server to become ready.
+     */
+    settleReadyWaiters(serverId, error) {
+        const serverData = this.servers.get(serverId);
+        if (!serverData?.readyWaiters?.length) {
+            return;
+        }
+        const waiters = serverData.readyWaiters;
+        serverData.readyWaiters = [];
+        for (const waiter of waiters) {
+            if (error) {
+                waiter.reject(error);
+            }
+            else {
+                waiter.resolve();
+            }
+        }
+    }
+    /**
+     * Start an external server process.
+     *
+     * Resolves once the MCP handshake AND tool discovery have completed — a
+     * resolved startServer() means the server's tools are in the registry.
+     * It used to resolve right after spawn, which let callers (agent join,
+     * restart) proceed against a server that had not finished — or would
+     * never finish — its handshake.
      */
     async startServer(serverId, agentId, channelId) {
         logger.info(`[START_SERVER] Starting server ${serverId}`);
@@ -302,8 +526,19 @@ export class ExternalMcpServerManager extends EventEmitter {
             logger.info(`[START_SERVER] Server ${serverId} already running`);
             return;
         }
+        if (status.status === 'starting') {
+            // Another caller is already starting this server — wait for that
+            // startup instead of spawning a second process.
+            logger.info(`[START_SERVER] Server ${serverId} already starting, awaiting readiness`);
+            await new Promise((resolve, reject) => {
+                serverData.readyWaiters = serverData.readyWaiters ?? [];
+                serverData.readyWaiters.push({ resolve, reject });
+            });
+            return;
+        }
         // Update status
         status.status = 'starting';
+        serverData.expectedExit = false;
         this.emitServerEvent(McpEvents.EXTERNAL_SERVER_SPAWN, serverId, agentId, channelId);
         try {
             // Spawn the process
@@ -351,31 +586,85 @@ export class ExternalMcpServerManager extends EventEmitter {
             const errorMessage = error instanceof Error ? error.message : String(error);
             logger.error(`❌ Failed to start server ${config.name}: ${errorMessage}`);
             this.handleServerError(serverId, errorMessage, agentId, channelId);
+            throw error instanceof Error ? error : new Error(errorMessage);
         }
+        // Wait for the handshake and tool discovery. Settled by
+        // initializeMcpConnection() on success, and by handleServerError()
+        // or the exit handler on failure — those already report the cause,
+        // so a rejection here propagates without being re-handled.
+        await new Promise((resolve, reject) => {
+            serverData.readyWaiters = serverData.readyWaiters ?? [];
+            serverData.readyWaiters.push({ resolve, reject });
+        });
     }
     /**
-     * Handle agent joining a channel - start channel servers and track connection
+     * Handle agent joining a channel — verify each channel server is actually
+     * alive before counting the agent as connected.
+     *
+     * This used to be a blind reference-count bump: a missing server record was
+     * silently skipped and a dead process was never probed, so agents "joined"
+     * servers that could not serve a single tool call, and the only downstream
+     * signal was a NOT FOUND warning when their allowlist resolved.
      */
     async onAgentJoinChannel(agentId, channelId) {
         logger.info(`Agent ${agentId} joining channel ${channelId} - checking for channel servers`);
         // Find all channel-scoped servers for this channel
         for (const [serverId, scopeData] of this.serverScopes.entries()) {
-            if (scopeData.scope === 'channel' && scopeData.scopeId === channelId) {
-                // Add agent to connected agents
-                scopeData.connectedAgents.add(agentId);
-                // Clear any pending keepAlive timer
+            if (scopeData.scope !== 'channel' || scopeData.scopeId !== channelId) {
+                continue;
+            }
+            const serverData = this.servers.get(serverId);
+            if (!serverData) {
+                // The production zombie state: a scope entry whose server record is
+                // gone. There is no config left to restart from, so remove the
+                // orphan loudly instead of pretending the agent connected.
+                logger.error(`Agent ${agentId} tried to join channel server ${serverId}, but its server record is gone ` +
+                    `(scope entry was orphaned). Removing the orphaned scope — the server must be re-registered.`);
                 if (scopeData.keepAliveTimer) {
                     clearTimeout(scopeData.keepAliveTimer);
-                    scopeData.keepAliveTimer = undefined;
                 }
-                // Start server if not already running
-                const serverData = this.servers.get(serverId);
-                if (serverData && serverData.status.status !== 'running') {
-                    logger.info(`Starting channel server ${serverId} for agent ${agentId}`);
+                this.serverScopes.delete(serverId);
+                continue;
+            }
+            try {
+                if (serverData.status.status !== 'running') {
+                    logger.info(`Starting channel server ${serverId} for agent ${agentId} (status: ${serverData.status.status})`);
                     await this.startServer(serverId);
                 }
-                logger.info(`Agent ${agentId} connected to channel server ${serverId} (${scopeData.connectedAgents.size} agents)`);
+                else {
+                    // Status says running — prove it. A live entry with a dead child
+                    // (or a wedged MCP connection) must trigger recovery at join
+                    // time, not a silent ref-count bump.
+                    const alive = serverData.process && !serverData.process.killed && serverData.process.exitCode === null;
+                    if (!alive) {
+                        logger.warn(`Channel server ${serverId} has no live process at agent join — restarting`);
+                        await this.restartServer(serverId);
+                    }
+                    else {
+                        try {
+                            await this.sendRequest(serverId, 'tools/list');
+                        }
+                        catch (probeError) {
+                            logger.warn(`Channel server ${serverId} did not answer the liveness probe at agent join ` +
+                                `(${probeError instanceof Error ? probeError.message : String(probeError)}) — restarting`);
+                            await this.restartServer(serverId);
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                logger.error(`Channel server ${serverId} could not be made available for agent ${agentId}: ` +
+                    `${error instanceof Error ? error.message : String(error)}`);
+                continue;
+            }
+            // Only a server that is verifiably up counts the agent as connected
+            scopeData.connectedAgents.add(agentId);
+            // Clear any pending keepAlive timer
+            if (scopeData.keepAliveTimer) {
+                clearTimeout(scopeData.keepAliveTimer);
+                scopeData.keepAliveTimer = undefined;
             }
+            logger.info(`Agent ${agentId} connected to channel server ${serverId} (${scopeData.connectedAgents.size} agents)`);
         }
     }
     /**
@@ -394,12 +683,18 @@ export class ExternalMcpServerManager extends EventEmitter {
                     const keepAliveMs = (scopeData.keepAliveMinutes || 5) * 60 * 1000;
                     logger.info(`Last agent left channel server ${serverId}, starting ${scopeData.keepAliveMinutes}min keepAlive timer`);
                     scopeData.keepAliveTimer = setTimeout(async () => {
+                        scopeData.keepAliveTimer = undefined;
+                        if (!this.servers.has(serverId)) {
+                            logger.warn(`KeepAlive expired for ${serverId}, but its server record is already gone — removing the orphaned scope`);
+                            this.serverScopes.delete(serverId);
+                            return;
+                        }
                         logger.info(`KeepAlive expired for server ${serverId}, stopping server`);
                         try {
-                            await this.stopServer(serverId);
+                            await this.stopServer(serverId, undefined, undefined, 'keepAlive expired');
                         }
                         catch (error) {
-                            logger.error(`Error stopping server ${serverId} after keepAlive:`, error);
+                            logger.error(`Error stopping server ${serverId} after keepAlive: ${error instanceof Error ? error.message : String(error)}`);
                         }
                     }, keepAliveMs);
                 }
@@ -411,24 +706,29 @@ export class ExternalMcpServerManager extends EventEmitter {
      */
     getServersByScope(scope, scopeId) {
         const servers = [];
-        for (const [serverId, scopeData] of this.serverScopes.entries()) {
-            if (scopeData.scope === scope) {
+        for (const [serverId, serverData] of this.servers.entries()) {
+            // Servers registered through the original global registration path
+            // predate serverScopes and are explicitly global, matching the tool
+            // snapshot contract in getAllExternalTools().
+            const scopeData = this.serverScopes.get(serverId);
+            const actualScope = scopeData?.scope ?? 'global';
+            if (actualScope === scope) {
                 // For channel/agent scope, match scopeId
-                if ((scope === 'channel' || scope === 'agent') && scopeData.scopeId !== scopeId) {
+                if ((scope === 'channel' || scope === 'agent') && scopeData?.scopeId !== scopeId) {
                     continue;
                 }
-                const serverData = this.servers.get(serverId);
-                if (serverData) {
-                    servers.push(serverData.status);
-                }
+                servers.push(serverData.status);
             }
         }
-        return servers;
+        return servers.sort((a, b) => a.id.localeCompare(b.id));
     }
     /**
-     * Stop an external server process
+     * Stop an external server process.
+     *
+     * An intentional stop: the exit handler will see `expectedExit` and will
+     * neither log the exit as a crash nor restart the process.
      */
-    async stopServer(serverId, agentId, channelId) {
+    async stopServer(serverId, agentId, channelId, reason) {
         const serverData = this.servers.get(serverId);
         if (!serverData) {
             throw new Error(`Server ${serverId} not found`);
@@ -437,6 +737,9 @@ export class ExternalMcpServerManager extends EventEmitter {
         if (status.status === 'stopped') {
             return;
         }
+        const droppedTools = status.tools.length;
+        logger.info(`Stopping server ${serverId} (${reason ?? 'no reason given'})` +
+            (droppedTools > 0 ? ` — removing its ${droppedTools} tool(s) from the registry` : ''));
         // Emit stop event
         this.emitServerEvent(McpEvents.EXTERNAL_SERVER_STOP, serverId, agentId, channelId);
         // Clear timers
@@ -451,14 +754,19 @@ export class ExternalMcpServerManager extends EventEmitter {
         // Fail anything still in flight before we kill the process, so callers get
         // a clear error rather than waiting out their own timeouts.
         this.rejectPendingRequests(serverId, 'server is stopping');
+        this.settleReadyWaiters(serverId, new Error(`Server ${serverId} was stopped (${reason ?? 'no reason given'})`));
         // Terminate process
         if (serverData.process) {
-            serverData.process.kill('SIGTERM');
-            // Force kill after timeout
-            setTimeout(() => {
-                if (serverData.process && !serverData.process.killed) {
+            serverData.expectedExit = true;
+            const stoppingProcess = serverData.process;
+            stoppingProcess.kill('SIGTERM');
+            // Force kill after timeout; the exit handler clears this when the
+            // process goes down on its own.
+            serverData.forceKillTimer = setTimeout(() => {
+                serverData.forceKillTimer = undefined;
+                if (!stoppingProcess.killed && stoppingProcess.exitCode === null) {
                     logger.warn(`Force killing server ${config.name}`);
-                    serverData.process.kill('SIGKILL');
+                    stoppingProcess.kill('SIGKILL');
                 }
             }, 5000);
         }
@@ -494,10 +802,15 @@ export class ExternalMcpServerManager extends EventEmitter {
     getAllExternalTools() {
         const tools = [];
         for (const [serverId, serverData] of this.servers) {
+            const scopeData = this.serverScopes.get(serverId);
+            const scope = scopeData?.scope ?? 'global';
+            const scopeId = scopeData?.scopeId;
             for (const tool of serverData.status.tools) {
                 tools.push({
                     ...tool,
-                    serverId
+                    serverId,
+                    scope,
+                    scopeId
                 });
             }
         }
@@ -513,29 +826,79 @@ export class ExternalMcpServerManager extends EventEmitter {
         const { config, status } = serverData;
         // Handle process exit
         process.on('exit', (code, signal) => {
+            // This process is down — its SIGKILL escalation timer is moot.
+            if (serverData.forceKillTimer) {
+                clearTimeout(serverData.forceKillTimer);
+                serverData.forceKillTimer = undefined;
+            }
+            const currentData = this.servers.get(serverId);
+            if (!currentData || currentData.process !== process) {
+                // Exit of a process instance that has already been replaced (restart)
+                // or whose server record is gone (unregistered). Not this record's
+                // state to change.
+                logger.debug(`Ignoring exit of a superseded process for ${serverId} (code ${code ?? 'null'}, signal ${signal ?? 'none'})`);
+                return;
+            }
+            const wasExpected = currentData.expectedExit === true;
+            currentData.expectedExit = false;
+            const droppedTools = status.tools.length;
             status.status = 'stopped';
             status.pid = undefined;
             // The handshake does not survive the process. A restarted server has to
             // perform it again before it can be marked running.
             status.initialized = false;
             status.initializing = false;
+            status.tools = [];
             // Anything still waiting on this process will never get a reply.
             this.rejectPendingRequests(serverId, `server exited (code ${code ?? 'null'}, signal ${signal ?? 'none'})`);
+            this.settleReadyWaiters(serverId, new Error(`Server ${serverId} exited during startup (code ${code ?? 'null'}, signal ${signal ?? 'none'})`));
             // Clear timers
             if (serverData.healthCheckTimer) {
                 clearInterval(serverData.healthCheckTimer);
                 serverData.healthCheckTimer = undefined;
             }
-            // Handle restart if configured
-            if (config.restartOnCrash && code !== 0 && status.restartCount < config.maxRestartAttempts) {
-                status.restartCount++;
-                setTimeout(() => {
-                    // Check if server still exists before attempting restart
-                    // (it may have been unregistered during the delay)
-                    if (this.servers.has(serverId)) {
-                        this.startServer(serverId);
-                    }
-                }, 2000); // Wait 2 seconds before restart
+            if (serverData.startupTimer) {
+                clearTimeout(serverData.startupTimer);
+                serverData.startupTimer = undefined;
+            }
+            if (wasExpected) {
+                logger.info(`Server ${serverId} exited after stop (code ${code ?? 'null'}, signal ${signal ?? 'none'})`);
+                return;
+            }
+            // Unexpected death. This used to happen in complete silence — no log
+            // line at any level — which is how production servers vanished from
+            // the tool registry with nothing to grep for.
+            logger.error(`Server ${serverId} exited unexpectedly (code ${code ?? 'null'}, signal ${signal ?? 'none'})` +
+                (droppedTools > 0 ? ` — its ${droppedTools} tool(s) are removed from the registry` : ''));
+            // Restart on ANY unexpected exit when configured — including a clean
+            // exit code. `code !== 0` used to gate this, leaving a child that
+            // exited 0 dead forever with no restart and no log.
+            if (config.restartOnCrash) {
+                if (status.restartCount < config.maxRestartAttempts) {
+                    status.restartCount++;
+                    logger.warn(`Restarting server ${serverId} in ${this.restartDelayMs}ms ` +
+                        `(attempt ${status.restartCount}/${config.maxRestartAttempts})`);
+                    setTimeout(() => {
+                        // Check if server still exists before attempting restart
+                        // (it may have been unregistered during the delay)
+                        if (this.servers.has(serverId)) {
+                            this.startServer(serverId).catch(error => {
+                                logger.error(`Restart of server ${serverId} failed: ` +
+                                    `${error instanceof Error ? error.message : String(error)}`);
+                            });
+                        }
+                    }, this.restartDelayMs);
+                }
+                else {
+                    // Out of restart budget: remove the server entirely rather than
+                    // leaving a zombie record + scope that agents can "join".
+                    this.removeServerAfterFailure(serverId, `crashed and exhausted its ${config.maxRestartAttempts} restart attempt(s)`);
+                    return;
+                }
+            }
+            else {
+                logger.error(`Server ${serverId} will not be restarted (restartOnCrash is off). ` +
+                    `An agent joining its channel will start it again on demand.`);
             }
             this.emitServerEvent(McpEvents.EXTERNAL_SERVER_STOPPED, serverId);
         });
@@ -642,7 +1005,7 @@ export class ExternalMcpServerManager extends EventEmitter {
         }
         const stdin = serverData.process.stdin;
         const requestId = uuidv4();
-        const timeoutMs = REQUEST_TIMEOUTS_MS[method];
+        const timeoutMs = this.requestTimeouts[method];
         return new Promise((resolve, reject) => {
             const timer = setTimeout(() => {
                 serverData.pending.delete(requestId);
@@ -735,12 +1098,27 @@ export class ExternalMcpServerManager extends EventEmitter {
         }
         try {
             await this.sendRequest(serverId, 'tools/list');
+            serverData.consecutiveHealthFailures = 0;
             this.emitServerHealthStatus(serverId, 'healthy');
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);
-            logger.warn(`Health check failed for ${serverId}: ${message}`);
+            serverData.consecutiveHealthFailures = (serverData.consecutiveHealthFailures ?? 0) + 1;
+            logger.warn(`Health check failed for ${serverId} ` +
+                `(${serverData.consecutiveHealthFailures} consecutive): ${message}`);
             this.emitServerHealthStatus(serverId, 'unhealthy');
+            // A process that is alive but no longer answering MCP is as dead as a
+            // crashed one — the exit handler will never fire for it. Recover here.
+            if (serverData.consecutiveHealthFailures >= HEALTH_FAILURES_BEFORE_RESTART &&
+                serverData.config.restartOnCrash &&
+                status.status === 'running') {
+                serverData.consecutiveHealthFailures = 0;
+                logger.error(`Server ${serverId} failed ${HEALTH_FAILURES_BEFORE_RESTART} consecutive health checks — restarting it`);
+                this.restartServer(serverId).catch(restartError => {
+                    logger.error(`Health-check restart of ${serverId} failed: ` +
+                        `${restartError instanceof Error ? restartError.message : String(restartError)}`);
+                });
+            }
         }
     }
     /**
@@ -859,6 +1237,14 @@ export class ExternalMcpServerManager extends EventEmitter {
         this.emitServerEvent(McpEvents.EXTERNAL_SERVER_STARTED, serverId);
         // Now that the connection is live, find out what the server offers.
         await this.discoverServerTools(serverId);
+        // A server that came up healthy earns back its full restart budget.
+        // restartCount used to only ever grow, so a server that crashed a few
+        // times over its lifetime — days apart, each recovered — permanently
+        // exhausted its budget and the next crash left it down for good.
+        serverData.status.restartCount = 0;
+        serverData.consecutiveHealthFailures = 0;
+        // Whoever awaited startServer() can proceed: the tools are discovered.
+        this.settleReadyWaiters(serverId);
     }
     /**
      * Ask a server what tools it has.
@@ -891,6 +1277,8 @@ export class ExternalMcpServerManager extends EventEmitter {
         status.status = 'error';
         status.lastError = error;
         logger.error(`❌ Server ${serverId} error: ${error}`);
+        // Anyone awaiting this server's startup gets the failure now.
+        this.settleReadyWaiters(serverId, new Error(`Server ${serverId} failed: ${error}`));
         // Emit error event
         this.emitServerErrorEvent(serverId, error, agentId, channelId);
     }
@@ -986,9 +1374,24 @@ export class ExternalMcpServerManager extends EventEmitter {
      * Cleanup and shutdown all servers
      */
     async shutdown() {
+        // Stop accepting lifecycle requests before child-process teardown. Node's
+        // removeAllListeners() below only affects this class's EventEmitter; it
+        // does not detach RxJS EventBus subscriptions.
+        for (const subscription of this.eventSubscriptions) {
+            subscription.unsubscribe();
+        }
+        this.eventSubscriptions = [];
         const shutdownPromises = Array.from(this.servers.keys()).map(serverId => this.stopServer(serverId));
         await Promise.allSettled(shutdownPromises);
+        // Clear pending keepAlive timers so nothing fires against cleared maps
+        for (const scopeData of this.serverScopes.values()) {
+            if (scopeData.keepAliveTimer) {
+                clearTimeout(scopeData.keepAliveTimer);
+            }
+        }
         this.servers.clear();
+        this.serverScopes.clear();
+        this.retiredChannelIds.clear();
         this.removeAllListeners();
     }
     /**
@@ -1044,22 +1447,20 @@ export class ExternalMcpServerManager extends EventEmitter {
         }
     }
     /**
-     * Restart a server by ID
+     * Restart a server by ID. Resolves after the restarted server has finished
+     * its handshake and tool discovery (see startServer). Throws when the
+     * restart fails, so callers can react instead of proceeding against a dead
+     * server.
      */
     async restartServer(serverId) {
-        try {
-            // Stop the server first
-            await this.stopServer(serverId);
-            // Wait a bit before restarting
-            await new Promise(resolve => setTimeout(resolve, 1000));
-            // Start the server again
-            await this.startServer(serverId);
-            return true;
-        }
-        catch (error) {
-            logger.error(`Failed to restart server ${serverId}: ${error instanceof Error ? error.message : String(error)}`);
-            return false;
-        }
+        logger.info(`Restarting server ${serverId}`);
+        // Stop the server first
+        await this.stopServer(serverId, undefined, undefined, 'restart');
+        // Give the old process a moment to release stdio
+        await new Promise(resolve => setTimeout(resolve, this.restartDelayMs));
+        // Start the server again — resolves after handshake + tool discovery
+        await this.startServer(serverId);
+        return true;
     }
     /**
      * Execute a tool on an external MCP server via JSON-RPC with auto-correction support